ruby-skill-bench 0.1.0

data/lib/skill_bench/cli/skill_command.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'optparse'
+
+module SkillBench
+  module Cli
+    # Handles the `skill-bench skill` subcommand.
+    # Parses options and delegates to Commands::SkillNew.
+    class SkillCommand
+      # Parses argv and executes the skill command.
+      #
+      # @param argv [Array<String>] Raw CLI arguments
+      # @return [Integer] Exit code
+      def self.call(argv)
+        new(argv).call
+      end
+
+      # @param argv [Array<String>] Raw CLI arguments
+      def initialize(argv)
+        @argv = argv
+      end
+
+      # Dispatches to the appropriate skill action.
+      #
+      def call
+        action = @argv.shift
+        case action
+        when 'new'
+          handle_new(@argv)
+        when '-h', '--help', 'help', nil
+          print_help
+          0
+        else
+          warn "Unknown skill action: #{action}"
+          1
+        end
+      end
+
+      private
+
+      def handle_new(argv)
+        options = { mode: 'simple', template: 'service_object' }
+        parser = OptionParser.new do |opts|
+          opts.banner = 'Usage: skill-bench skill new <name> [options]'
+          opts.on('--mode MODE', 'simple, advanced, or rails') { |v| options[:mode] = v }
+          opts.on('--template TYPE', 'service_object, concern, active_record_model') { |v| options[:template] = v }
+          opts.on('-h', '--help', 'Prints this help') do
+            puts opts
+            raise SkillBench::HelpRequested
+          end
+        end
+        parser.parse!(argv)
+
+        name = argv.shift
+        return error_missing_name unless name
+
+        Commands::SkillNew.run(name: name, **options)
+        puts "Created skill: #{name}"
+        0
+      rescue SkillBench::HelpRequested
+        0
+      rescue StandardError => e
+        warn "Error: #{e.message}"
+        1
+      end
+
+      def print_help
+        puts 'Usage: skill-bench skill new <name> [options]'
+        puts '  --mode MODE        simple, advanced, or rails (default: simple)'
+        puts '  --template TYPE    service_object, concern, active_record_model (default: service_object)'
+      end
+
+      def error_missing_name
+        warn 'Error: skill name is required'
+        1
+      end
+    end
+  end
+end

data/lib/skill_bench/cli.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative 'cli/init_command'
+require_relative 'cli/run_command'
+require_relative 'cli/skill_command'
+require_relative 'cli/eval_command'
+require_relative 'cli/help_printer'
+require_relative 'cli/result_printer'
+
+module SkillBench
+  # Raised when -h/--help is passed to abort OptionParser and return exit code 0.
+  class HelpRequested < StandardError; end
+
+  # Thin CLI dispatcher that routes subcommands to their handlers.
+  class CLI
+    # Entry point called from bin/skill-bench.
+    #
+    # @param argv [Array<String>] Raw CLI arguments.
+    # @return [Integer] Exit code.
+    def self.call(argv)
+      new(argv).call
+    end
+
+    # @param argv [Array<String>] Raw CLI arguments.
+    def initialize(argv)
+      @argv = argv
+    end
+
+    # Dispatches to the appropriate subcommand handler.
+    #
+    # @return [Integer] Exit code.
+    def call
+      help = -> { Cli::HelpPrinter.call }
+      return help.call if @argv.empty?
+
+      subcommand = @argv.shift
+      case subcommand
+      when 'init'  then Cli::InitCommand.call(@argv)
+      when 'run'   then Cli::RunCommand.call(@argv)
+      when 'skill' then Cli::SkillCommand.call(@argv)
+      when 'eval'  then Cli::EvalCommand.call(@argv)
+      when '-h', '--help', 'help'
+        help.call
+      else
+        warn "Unknown subcommand: #{subcommand}"
+        warn "Run 'skill-bench help' for usage."
+        1
+      end
+    end
+  end
+end

data/lib/skill_bench/client.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative 'clients/all'
+
+module SkillBench
+  # Facade for calling LLM clients.
+  # Delegates to the configured provider.
+  class Client
+    # Calls the configured LLM provider with the given parameters.
+    #
+    # @param system_prompt [String] System prompt for the LLM
+    # @param messages [Array<Hash>] Conversation messages
+    # @param provider [Symbol, nil] Override the configured LLM provider (e.g., :deepseek, :openai)
+    # @param options [Hash] Provider-specific options (api_key, model, etc.)
+    # @return [Hash] Response from the LLM
+    def self.call(system_prompt:, messages:, provider: nil, **options)
+      resolved = provider || Config.current_llm_provider || :openai
+      client_class = Clients::ProviderRegistry.for(resolved)
+      warn "WARNING: LLM provider '#{resolved}' is not configured. Falling back to null client." if client_class == Clients::Providers::NullClient
+      client_class.call(system_prompt: system_prompt, messages: messages, **options)
+    end
+  end
+end

data/lib/skill_bench/clients/all.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative 'response_parser'
+require_relative 'response_error_handler'
+require_relative 'request_builder'
+require_relative 'retry_handler'
+require_relative 'base_client'
+require_relative 'provider_config'
+require_relative 'provider_registry'
+require_relative 'providers/null_client'
+require_relative 'providers/anthropic'
+require_relative 'providers/openai'
+require_relative 'providers/gemini'
+require_relative 'providers/ollama'
+require_relative 'providers/azure_openai'
+require_relative 'providers/opencode'
+require_relative 'providers/groq'
+require_relative 'providers/deepseek'
+require_relative 'providers/openrouter'

data/lib/skill_bench/clients/base_client.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+require_relative '../config'
+require_relative 'provider_config'
+require_relative 'response_parser'
+require_relative 'response_error_handler'
+require_relative 'request_builder'
+require_relative 'retry_handler'
+
+module SkillBench
+  module Clients
+    # Base class for all LLM provider clients.
+    # Orchestrates request execution, response parsing, and error handling.
+    # Following the Template Method pattern and ruby-service-objects standards.
+    class BaseClient
+      attr_reader :messages, :system_prompt, :tools, :api_key, :model, :options
+
+      # Standard entry point for the service object.
+      #
+      # @param system_prompt [String] The system instruction for the LLM.
+      # @param messages [Array<Hash>] The list of conversation messages.
+      # @param tools [Array<Hash>] (optional) Array of tool definitions.
+      # @param options [Hash] (optional) Additional provider-specific options.
+      # @return [Hash] with :success [Boolean] and :response [Hash] keys.
+      def self.call(system_prompt:, messages:, tools: [], **options)
+        new(system_prompt: system_prompt, messages: messages, tools: tools, **options).call
+      end
+
+      # Initializes the client with validated parameters.
+      # @param options [Hash] Configuration overrides.
+      def initialize(options = {})
+        config = ProviderConfig.call(provider: provider_name, options: options)
+
+        @api_key = config[:api_key]
+        @model = config[:model]
+        @base_url_config = config[:base_url]
+        @request_path_config = config[:request_path]
+        @provider_display_name = config[:provider_name]
+
+        @location = config[:location]
+        @project_id = config[:project_id]
+        @endpoint = config[:endpoint]
+        @api_version = config[:api_version]
+
+        @system_prompt = options[:system_prompt] || ''
+        @messages = options[:messages] || []
+        @tools = options[:tools] || []
+      end
+
+      # Abstract method to return the provider identifier.
+      #
+      # @return [Symbol]
+      def provider_name
+        raise NotImplementedError, "#{self.class} must implement #provider_name"
+      end
+
+      # Sends the request to the LLM and returns the standardized response.
+      #
+      # @return [Hash] standardized response with success, body, and usage information.
+      def call
+        return config_error unless valid_config?
+
+        response = execute_request
+        handle_response(response)
+      rescue Faraday::Error => e
+        ResponseErrorHandler.handle_exception(e, 'Network Error')
+      rescue JSON::ParserError => e
+        ResponseErrorHandler.handle_exception(e, 'Parsing Error')
+      rescue StandardError => e
+        ResponseErrorHandler.handle_exception(e, 'Unexpected Error')
+      end
+
+      protected
+
+      # Returns the base URL for the LLM API.
+      #
+      # @return [String]
+      def base_url
+        @base_url_config || raise(NotImplementedError, "#{self.class} must implement #base_url")
+      end
+
+      # Returns the request path for the LLM API.
+      #
+      # @return [String]
+      def request_path
+        @request_path_config || raise(NotImplementedError, "#{self.class} must implement #request_path")
+      end
+
+      # @return [Hash]
+      def request_headers
+        {
+          'Authorization' => "Bearer #{@api_key}",
+          'Content-Type' => 'application/json'
+        }
+      end
+
+      # @return [Hash]
+      def request_body
+        body = {
+          model: model_name,
+          messages: [{ role: 'system', content: @system_prompt }] + @messages
+        }
+        body[:tools] = @tools if @tools&.any?
+        body
+      end
+
+      # @return [String]
+      def model_name
+        @model
+      end
+
+      # Validates that the configuration is complete.
+      #
+      # @return [Boolean]
+      def valid_config?
+        missing_config_keys.empty?
+      end
+
+      # Returns the list of configuration keys that are required but missing.
+      #
+      # @return [Array<String>]
+      def missing_config_keys
+        missing = []
+        missing << 'API Key' if @api_key.nil? || @api_key.empty?
+        missing
+      end
+
+      # Standardized error response when configuration is missing.
+      #
+      # @return [Hash]
+      def config_error
+        missing = missing_config_keys
+        message = if missing.length > 1
+                    "#{missing[0...-1].join(', ')}, and #{missing[-1]} not set for #{@provider_display_name}"
+                  else
+                    "#{missing.first} not set for #{@provider_display_name}"
+                  end
+        { success: false, response: { error: { message: message } }, result: message, status: 'error' }
+      end
+
+      # Extracts the message hash from the provider's specific response body structure.
+      # Default implementation for OpenAI-compatible providers.
+      #
+      # @param body [Hash]
+      # @return [Hash, nil]
+      def extract_message(body)
+        ResponseParser.extract_openai_message(body)
+      end
+
+      # Extracts token usage from the provider-specific response.
+      # @param body [Hash]
+      # @return [Hash]
+      def extract_usage(body)
+        ResponseParser.extract_openai_usage(body)
+      end
+
+      private
+
+      def execute_request
+        RetryHandler.call do
+          connection = RequestBuilder.build_connection(base_url)
+          RequestBuilder.execute(connection, request_path, headers: request_headers, body: request_body)
+        end
+      end
+
+      def handle_response(response)
+        parsed = ResponseParser.parse_body(response)
+        return failure_response(response, parsed) unless response.success?
+
+        body_error = parsed.is_a?(Hash) ? (parsed[:error] || parsed['error']) : nil
+        if body_error
+          error_msg = body_error.is_a?(Hash) ? (body_error[:message] || body_error['message']) : body_error.to_s
+          return {
+            success: false,
+            result: "API Error: #{error_msg}",
+            usage: extract_usage(parsed),
+            response: { error: { message: "API Error: #{error_msg}" } },
+            status: 'error'
+          }
+        end
+
+        message = extract_message(parsed)
+        return missing_message_response(response, parsed) unless ResponseParser.valid_message?(message)
+
+        success_response(parsed, message)
+      end
+
+      def success_response(parsed, message)
+        content = ResponseParser.extract_content(message)
+        {
+          success: true,
+          result: content,
+          usage: extract_usage(parsed),
+          response: parsed.merge(message: message),
+          status: 'success'
+        }
+      end
+
+      def failure_response(response, parsed)
+        ResponseErrorHandler.failure_response(response, parsed) { |body| extract_usage(body) }
+      end
+
+      def missing_message_response(response, parsed)
+        SkillBench::ErrorLogger.log_error(
+          StandardError.new("LLM response missing message content. Response keys: #{parsed.is_a?(Hash) ? parsed.keys.inspect : parsed.class}"),
+          'BaseClient'
+        )
+        ResponseErrorHandler.missing_message_response(response, parsed) { |body| extract_usage(body) }
+      end
+    end
+  end
+end

data/lib/skill_bench/clients/provider_config.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative '../config'
+
+module SkillBench
+  module Clients
+    # Service object to load and validate provider configuration.
+    class ProviderConfig
+      # @param provider [Symbol] provider identifier (e.g., :openai, :ollama)
+      # @param options [Hash] override options
+      # @return [Hash] standardized configuration
+      def self.call(provider:, options: {})
+        new(provider, options).call
+      end
+
+      def initialize(provider, options)
+        @provider = provider.to_sym
+        @options = options
+        @config = SkillBench::Config.for_provider(@provider) || {}
+      end
+
+      # Loads and returns standardized provider configuration.
+      #
+      # @return [Hash] Standardized configuration with api_key, model, base_url, etc.
+      def call
+        {
+          api_key: fetch_config(:api_key),
+          model: fetch_config(:model),
+          base_url: fetch_config(:base_url),
+          request_path: fetch_config(:request_path),
+          provider_name: @provider.to_s.capitalize,
+          # Provider-specific extras (nil when not present)
+          endpoint: fetch_config(:endpoint),
+          location: fetch_config(:location),
+          project_id: fetch_config(:project_id),
+          api_version: fetch_config(:api_version)
+        }
+      end
+
+      private
+
+      def fetch_config(key)
+        @options[key] || @config[key]
+      end
+    end
+  end
+end

data/lib/skill_bench/clients/provider_registry.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative 'providers/null_client'
+require_relative '../error_logger'
+
+module SkillBench
+  module Clients
+    # ProviderRegistry manages registration and lookup of LLM provider classes.
+    # Follows the Registry pattern for extensible provider discovery.
+    class ProviderRegistry
+      class << self
+        # Registers a provider class with a given name.
+        #
+        # @param name [Symbol] the provider identifier (e.g., :openai, :gemini)
+        # @param klass [Class] the provider class implementing the client interface
+        # @return [void]
+        def register(name, klass)
+          providers[name] = klass
+        end
+
+        # Looks up a provider class by name.
+        # Returns NullClient if the provider is not registered (with a warning).
+        #
+        # @param name [Symbol] the provider identifier
+        # @return [Class] the provider class or NullClient
+        def for(name)
+          providers.fetch(name) do
+            SkillBench::ErrorLogger.log_error(
+              StandardError.new("Unknown provider '#{name}', falling back to NullClient"),
+              'ProviderRegistry Warning'
+            )
+            Providers::NullClient
+          end
+        end
+
+        # Looks up a provider class by name, raising if not found.
+        #
+        # @param name [Symbol] the provider identifier
+        # @return [Class] the provider class
+        # @raise [ArgumentError] if the provider is not registered
+        def for!(name)
+          providers.fetch(name) do
+            raise ArgumentError, "Unknown provider '#{name}'. Registered: #{providers.keys.join(', ')}"
+          end
+        end
+
+        # Returns all registered providers.
+        #
+        # @return [Hash] mapping of provider names to classes
+        def providers
+          @providers ||= {}
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/clients/provider_schemas.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Clients
+    # Registry of provider configuration schemas with default placeholder values.
+    # Each provider defines its required configuration keys and sensible defaults.
+    class ProviderSchemas
+      PROVIDER_SCHEMAS = {
+        openai: {
+          api_key: nil,
+          model: 'gpt-4o'
+        }.freeze,
+        anthropic: {
+          api_key: nil,
+          model: 'claude-opus-4-7'
+        }.freeze,
+        gemini: {
+          api_key: nil,
+          model: 'gemini-1.5-flash-latest',
+          location: 'us-central1',
+          project_id: nil
+        }.freeze,
+        ollama: {
+          api_key: nil,
+          model: 'qwen:7b',
+          base_url: nil
+        }.freeze,
+        azure: {
+          api_key: nil,
+          model: 'gpt-4',
+          endpoint: nil,
+          api_version: nil
+        }.freeze,
+        groq: {
+          api_key: nil,
+          model: 'llama-3.3-70b-versatile'
+        }.freeze,
+        deepseek: {
+          api_key: nil,
+          model: 'deepseek-chat'
+        }.freeze,
+        opencode: {
+          api_key: nil,
+          model: 'opencode-model',
+          base_url: nil
+        }.freeze,
+        openrouter: {
+          api_key: nil,
+          model: 'anthropic/claude-3.5-sonnet'
+        }.freeze
+      }.freeze
+
+      # Returns the configuration schema for a given provider.
+      #
+      # @param provider [Symbol] Provider name
+      # @return [Hash] Provider configuration schema with placeholder values
+      # @raise [ArgumentError] if provider is not registered
+      def self.for(provider)
+        schema = PROVIDER_SCHEMAS.fetch(provider) do
+          raise(ArgumentError, "Unknown provider: #{provider}. Available: #{PROVIDER_SCHEMAS.keys.join(', ')}")
+        end
+        schema.dup
+      end
+
+      # Returns list of all registered provider names.
+      #
+      # @return [Array<Symbol>] Provider names
+      def self.names
+        PROVIDER_SCHEMAS.keys
+      end
+    end
+  end
+end