ruby-skill-bench 0.1.0

data/lib/skill_bench/clients/providers/anthropic.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+require_relative '../base_client'
+require_relative '../provider_registry'
+
+module SkillBench
+  module Clients
+    module Providers
+      # Anthropic Claude-specific LLM client.
+      # Uses the Messages API endpoint with Claude models.
+      class Anthropic < BaseClient
+        SkillBench::Clients::ProviderRegistry.register(:anthropic, self)
+
+        VERSION = '2023-06-01'
+
+        # Returns the provider identifier.
+        #
+        # @return [Symbol]
+        def provider_name
+          :anthropic
+        end
+
+        protected
+
+        # Returns the base URL for Anthropic API.
+        #
+        # @return [String]
+        def base_url
+          @base_url_config || 'https://api.anthropic.com'
+        end
+
+        # Returns the request path for the Messages API.
+        #
+        # @return [String]
+        def request_path
+          @request_path_config || '/v1/messages'
+        end
+
+        # Returns the headers required for Anthropic API.
+        #
+        # @return [Hash]
+        def request_headers
+          {
+            'x-api-key' => @api_key,
+            'anthropic-version' => VERSION,
+            'Content-Type' => 'application/json'
+          }
+        end
+
+        # Standardizes the request body for Anthropic's Messages API.
+        #
+        # @return [Hash]
+        def request_body
+          body = {
+            model: @model,
+            max_tokens: 4096,
+            system: @system_prompt,
+            messages: translate_messages(@messages)
+          }
+          body[:tools] = self.class.translate_tools(@tools) if @tools&.any?
+          body
+        end
+
+        private
+
+        def extract_message(body)
+          content_blocks = body[:content] || body['content']
+          return default_message unless content_blocks.is_a?(Array)
+
+          grouped = content_blocks.group_by { |block| block_type(block) }
+          message = build_base_message(grouped)
+          add_tool_calls(message, grouped) if grouped['tool_use']&.any?
+          message
+        end
+
+        def default_message
+          { 'role' => 'assistant', 'content' => '' }
+        end
+
+        def block_type(block)
+          (block[:type] || block['type']).to_s
+        end
+
+        def build_base_message(grouped)
+          text_block = grouped['text']&.first
+          {
+            'role' => 'assistant',
+            'content' => extract_text(text_block)
+          }
+        end
+
+        def extract_text(text_block)
+          return '' unless text_block
+
+          text_block[:text] || text_block['text'] || ''
+        end
+
+        def add_tool_calls(message, grouped)
+          message['tool_calls'] = grouped['tool_use'].map do |block|
+            {
+              'id' => block[:id] || block['id'],
+              'type' => 'function',
+              'function' => {
+                'name' => block[:name] || block['name'],
+                'arguments' => (block[:input] || block['input']).to_json
+              }
+            }
+          end
+        end
+
+        # Extracts token usage from Anthropic's response.
+        #
+        # @param body [Hash]
+        # @return [Hash]
+        def extract_usage(body)
+          usage = body[:usage] || body['usage'] || {}
+          input = usage[:input_tokens] || usage['input_tokens'] || 0
+          output = usage[:output_tokens] || usage['output_tokens'] || 0
+          {
+            prompt_tokens: input,
+            completion_tokens: output,
+            total_tokens: input + output
+          }
+        end
+
+        # Translates a list of messages to Anthropic's expected format.
+        # Handles user, assistant, and tool result message types.
+        #
+        # @param messages [Array<Hash>] List of standardized messages.
+        # @return [Array<Hash>] List of messages formatted for Anthropic.
+        def translate_messages(messages)
+          messages.map { |msg| translate_single_message(msg) }
+        end
+
+        # Translates a single message to Anthropic format.
+        def translate_single_message(msg)
+          klass = self.class
+          role = (msg[:role] || msg['role']).to_s
+          case role
+          when 'assistant' then klass.translate_assistant_message(msg)
+          when 'tool'      then klass.translate_tool_message(msg)
+          else
+            { role: role, content: msg[:content] || msg['content'] }
+          end
+        end
+
+        class << self
+          # Translates standard tool definitions to Anthropic tool format.
+          #
+          # @param tools [Array<Hash>] List of tool definitions.
+          # @return [Array<Hash>] Translated tools for Anthropic.
+          def translate_tools(tools)
+            tools.map do |tool|
+              {
+                name: tool.dig(:function, :name) || tool.dig('function', 'name'),
+                description: tool.dig(:function, :description) || tool.dig('function', 'description'),
+                input_schema: tool.dig(:function, :parameters) || tool.dig('function', 'parameters')
+              }
+            end
+          end
+
+          # Translates assistant message with tool calls to Anthropic format.
+          #
+          # @param msg [Hash] The assistant message.
+          # @return [Hash] Translated message for Anthropic.
+          def translate_assistant_message(msg)
+            content = []
+            text = msg[:content] || msg['content']
+            content << { type: 'text', text: text } if text && !text.empty?
+
+            (msg[:tool_calls] || msg['tool_calls'])&.each do |tool_call|
+              content << build_tool_use_block(tool_call)
+            end
+
+            { role: 'assistant', content: content }
+          end
+
+          # Translates tool result message to Anthropic format.
+          #
+          # @param msg [Hash] The tool result message.
+          # @return [Hash] Translated message for Anthropic.
+          def translate_tool_message(msg)
+            {
+              role: 'user',
+              content: [
+                {
+                  type: 'tool_result',
+                  tool_use_id: msg[:tool_call_id] || msg['tool_call_id'],
+                  content: msg[:content] || msg['content']
+                }
+              ]
+            }
+          end
+
+          private
+
+          def build_tool_use_block(tool_call)
+            {
+              type: 'tool_use',
+              id: tool_call[:id] || tool_call['id'],
+              name: tool_call.dig(:function, :name) || tool_call.dig('function', 'name'),
+              input: parse_tool_arguments(tool_call.dig(:function, :arguments) || tool_call.dig('function', 'arguments'))
+            }
+          end
+
+          def parse_tool_arguments(args_raw)
+            return nil if args_raw.nil?
+            return args_raw if args_raw.is_a?(Hash)
+            return nil unless args_raw.is_a?(String)
+
+            JSON.parse(args_raw)
+          rescue JSON::ParserError
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/clients/providers/azure_openai.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'cgi'
+require_relative '../base_client'
+require_relative '../provider_registry'
+
+module SkillBench
+  module Clients
+    module Providers
+      # Azure OpenAI provider using the OpenAI-compatible API.
+      #
+      # This provider bridges the gap between standard OpenAI requests and Azure's
+      # deployment-based endpoint structure.
+      #
+      # @see https://learn.microsoft.com/en-us/azure/ai-services/openai/reference
+      class AzureOpenAI < BaseClient
+        SkillBench::Clients::ProviderRegistry.register(:azure, self)
+
+        # Default API version if none is provided.
+        DEFAULT_API_VERSION = '2024-02-15-preview'
+
+        # Returns the provider identifier.
+        #
+        # @return [Symbol]
+        def provider_name
+          :azure
+        end
+
+        protected
+
+        # Returns the base URL for Azure OpenAI.
+        #
+        # @return [String]
+        def base_url
+          @endpoint.to_s
+        end
+
+        # Returns the request path including the deployment name and API version.
+        #
+        # @return [String]
+        def request_path
+          api_ver = @api_version || DEFAULT_API_VERSION
+          "/openai/deployments/#{CGI.escape(@model.to_s)}/chat/completions?api-version=#{CGI.escape(api_ver.to_s)}"
+        end
+
+        # Returns the headers required for Azure OpenAI authentication.
+        #
+        # @return [Hash]
+        def request_headers
+          {
+            'api-key' => @api_key,
+            'Content-Type' => 'application/json'
+          }
+        end
+
+        private
+
+        # @return [Array<String>]
+        def missing_config_keys
+          missing = []
+          missing << 'AZURE_OPENAI_API_KEY'  if @api_key.to_s.strip.empty?
+          missing << 'AZURE_OPENAI_ENDPOINT' if @endpoint.to_s.strip.empty?
+          missing << 'AZURE_OPENAI_MODEL'    if @model.to_s.strip.empty?
+          missing
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/clients/providers/deepseek.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative '../base_client'
+require_relative '../provider_registry'
+
+module SkillBench
+  module Clients
+    module Providers
+      # DeepSeek-specific LLM client.
+      # Uses OpenAI-compatible chat completions API.
+      class DeepSeek < BaseClient
+        SkillBench::Clients::ProviderRegistry.register(:deepseek, self)
+
+        # Returns the provider identifier.
+        #
+        # @return [Symbol]
+        def provider_name
+          :deepseek
+        end
+
+        protected
+
+        # Returns the base URL for DeepSeek API.
+        #
+        # @return [String]
+        def base_url
+          @base_url_config || 'https://api.deepseek.com'
+        end
+
+        # Returns the request path for chat completions.
+        #
+        # @return [String]
+        def request_path
+          @request_path_config || '/chat/completions'
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/clients/providers/gemini.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'cgi'
+require_relative '../base_client'
+require_relative '../provider_registry'
+
+module SkillBench
+  module Clients
+    module Providers
+      # Google Gemini provider using the OpenAI-compatible Vertex AI endpoint.
+      #
+      # This client handles the authentication and routing for Google's Vertex AI
+      # OpenAI-compatible API, allowing it to be used alongside other providers.
+      #
+      # @see https://cloud.google.com/vertex-ai/docs/generative-ai/model-reference/openai-compatible-api
+      class Gemini < BaseClient
+        SkillBench::Clients::ProviderRegistry.register(:gemini, self)
+
+        # Returns the provider identifier.
+        #
+        # @return [Symbol]
+        def provider_name
+          :gemini
+        end
+
+        protected
+
+        # Returns the base URL for Gemini's Vertex AI endpoint.
+        #
+        # @return [String]
+        def base_url
+          "https://#{CGI.escape(@location.to_s)}-aiplatform.googleapis.com"
+        end
+
+        # Returns the request path for the Vertex AI OpenAI-compatible endpoint.
+        #
+        # @return [String]
+        def request_path
+          "/v1/projects/#{CGI.escape(@project_id.to_s)}/locations/#{CGI.escape(@location.to_s)}/endpoints/openapi/chat/completions"
+        end
+
+        # Model name formatted for Vertex AI.
+        #
+        # @return [String]
+        def model_name
+          "google/#{@model}"
+        end
+
+        private
+
+        # @return [Array<String>]
+        def missing_config_keys
+          missing = []
+          missing << 'GEMINI_API_KEY'     if @api_key.to_s.strip.empty?
+          missing << 'GEMINI_PROJECT_ID'  if @project_id.to_s.strip.empty?
+          missing << 'GEMINI_LOCATION'    if @location.to_s.strip.empty?
+          missing << 'GEMINI_MODEL'       if @model.to_s.strip.empty?
+          missing
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/clients/providers/groq.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative '../base_client'
+require_relative '../provider_registry'
+
+module SkillBench
+  module Clients
+    module Providers
+      # Groq-specific LLM client.
+      # Uses OpenAI-compatible chat completions API.
+      class Groq < BaseClient
+        SkillBench::Clients::ProviderRegistry.register(:groq, self)
+
+        # Returns the provider identifier.
+        #
+        # @return [Symbol]
+        def provider_name
+          :groq
+        end
+
+        protected
+
+        # Returns the base URL for Groq API.
+        #
+        # @return [String]
+        def base_url
+          @base_url_config || 'https://api.groq.com/openai/v1'
+        end
+
+        # Returns the request path for chat completions.
+        #
+        # @return [String]
+        def request_path
+          @request_path_config || '/chat/completions'
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/clients/providers/null_client.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require_relative '../base_client'
+
+module SkillBench
+  module Clients
+    module Providers
+      # Null Object implementation for unsupported LLM providers.
+      # Extends BaseClient for interface consistency.
+      class NullClient < BaseClient
+        # Returns the provider identifier.
+        #
+        # @return [Symbol]
+        def provider_name
+          :null
+        end
+
+        protected
+
+        # Always returns an empty string for the base URL.
+        #
+        # @return [String]
+        def base_url
+          ''
+        end
+
+        # Always returns an empty string for the request path.
+        #
+        # @return [String]
+        def request_path
+          ''
+        end
+
+        # Standardized error response for unsupported providers.
+        #
+        # @return [Hash]
+        def config_error
+          provider = SkillBench::Config.current_llm_provider
+          { success: false, response: { error: { message: "Unsupported or unconfigured LLM provider: '#{provider}'" } } }
+        end
+
+        # NullClient is never valid - always returns config error.
+        # @return [false]
+        def valid_config?
+          false
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/clients/providers/ollama.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative '../base_client'
+require_relative '../provider_registry'
+
+module SkillBench
+  module Clients
+    module Providers
+      # Ollama-specific LLM client.
+      # Extends BaseClient to interact with an Ollama server (commonly used for open‑source models such as Qwen 3.5).
+      # Ollama does not require an API key but requires a model to be configured.
+      class Ollama < BaseClient
+        SkillBench::Clients::ProviderRegistry.register(:ollama, self)
+
+        # Returns the provider identifier.
+        #
+        # @return [Symbol]
+        def provider_name
+          :ollama
+        end
+
+        protected
+
+        # Returns the base URL for Ollama service.
+        # Checks the OLLAMA_BASE_URL env var, then the evaluator config, then falls back to localhost.
+        #
+        # @return [String]
+        def base_url
+          env_url = ENV.fetch('OLLAMA_BASE_URL', nil)
+          return env_url unless env_url.to_s.empty?
+
+          config_url = SkillBench::Config.llm_providers_config.dig(:ollama, :base_url)
+          return config_url unless config_url.to_s.empty?
+
+          'http://localhost:11434'
+        end
+
+        # Returns the request path for chat completions.
+        #
+        # @return [String]
+        def request_path
+          '/v1/chat/completions'
+        end
+
+        # Returns headers for the request. Authorization is included only when an API key is present.
+        #
+        # @return [Hash]
+        def request_headers
+          headers = { 'Content-Type' => 'application/json' }
+          headers['Authorization'] = "Bearer #{@api_key}" if @api_key && !@api_key.to_s.empty?
+          headers
+        end
+
+        # Ollama only requires a model; an API key is optional.
+        #
+        # @return [Array<String>]
+        def missing_config_keys
+          @model.to_s.strip.empty? ? ['OLLAMA_MODEL'] : []
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/clients/providers/openai.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require_relative '../base_client'
+require_relative '../provider_registry'
+
+module SkillBench
+  module Clients
+    module Providers
+      # OpenAI-specific LLM client.
+      # Inherits common logic from BaseClient.
+      class OpenAI < BaseClient
+        SkillBench::Clients::ProviderRegistry.register(:openai, self)
+
+        # Returns the provider identifier.
+        #
+        # @return [Symbol]
+        def provider_name
+          :openai
+        end
+
+        protected
+
+        # Returns the base URL for OpenAI API.
+        #
+        # @return [String]
+        def base_url
+          @base_url_config || 'https://api.openai.com'
+        end
+
+        # Returns the request path for chat completions.
+        #
+        # @return [String]
+        def request_path
+          @request_path_config || '/v1/chat/completions'
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/clients/providers/opencode.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require_relative '../base_client'
+require_relative '../provider_registry'
+
+module SkillBench
+  module Clients
+    module Providers
+      # OpenCode provider client.
+      #
+      # IMPORTANT: OpenCode does not host a public LLM API. This provider is a
+      # thin wrapper around an OpenAI-compatible endpoint that YOU provide (e.g.
+      # LiteLLM proxy, vLLM, or a company gateway). You MUST set `base_url` in
+      # `skill-bench.json` or via the `SKILL_BENCH_OPENCODE_BASE_URL` environment
+      # variable, otherwise the provider will fail with "Base URL not set for Opencode".
+      class OpenCode < BaseClient
+        SkillBench::Clients::ProviderRegistry.register(:opencode, self)
+
+        # Returns the provider identifier.
+        #
+        # @return [Symbol]
+        def provider_name
+          :opencode
+        end
+
+        protected
+
+        # Returns the base URL for OpenCode API.
+        # OpenCode does not host a public LLM endpoint; users must configure
+        # a custom base_url (e.g. a self-hosted OpenAI-compatible proxy).
+        #
+        # @return [String]
+        def base_url
+          @base_url_config
+        end
+
+        # Returns the request path for chat completions.
+        #
+        # @return [String]
+        def request_path
+          @request_path_config || '/v1/chat/completions'
+        end
+
+        private
+
+        # @return [Array<String>]
+        def missing_config_keys
+          missing = []
+          missing << 'API Key' if @api_key.nil? || @api_key.empty?
+          missing << 'Base URL' if @base_url_config.nil? || @base_url_config.to_s.empty?
+          missing
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/clients/providers/openrouter.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative '../base_client'
+require_relative '../provider_registry'
+
+module SkillBench
+  module Clients
+    module Providers
+      # OpenRouter LLM client.
+      # Uses OpenRouter's OpenAI-compatible API to access multiple model providers.
+      # Inherits common logic from BaseClient.
+      class OpenRouter < BaseClient
+        SkillBench::Clients::ProviderRegistry.register(:openrouter, self)
+
+        # Returns the provider identifier.
+        #
+        # @return [Symbol]
+        def provider_name
+          :openrouter
+        end
+
+        protected
+
+        # Returns the base URL for OpenRouter API.
+        #
+        # @return [String]
+        def base_url
+          @base_url_config || 'https://openrouter.ai'
+        end
+
+        # Returns the request path for chat completions.
+        #
+        # @return [String]
+        def request_path
+          @request_path_config || '/api/v1/chat/completions'
+        end
+      end
+    end
+  end
+end