ruby-skill-bench 0.1.0

data/lib/skill_bench/config/env_overrides.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module SkillBench
+  class Config
+    # Builds configuration overrides from evaluator environment variables.
+    class EnvOverrides
+      # Mapping from environment variable names to provider configuration keys.
+      # Supports both prefixed (SKILL_BENCH_*) and unprefixed variants for
+      # backward compatibility. Prefixed variants are the documented standard.
+      ENV_TO_PROVIDER_SETTINGS = {
+        'SKILL_BENCH_OPENAI_API_KEY' => %i[openai api_key],
+        'OPENAI_API_KEY' => %i[openai api_key],
+        'SKILL_BENCH_ANTHROPIC_API_KEY' => %i[anthropic api_key],
+        'ANTHROPIC_API_KEY' => %i[anthropic api_key],
+        'SKILL_BENCH_OPENAI_MODEL' => %i[openai model],
+        'OPENAI_MODEL' => %i[openai model],
+        'SKILL_BENCH_OPENAI_BASE_URL' => %i[openai base_url],
+        'OPENAI_BASE_URL' => %i[openai base_url],
+        'SKILL_BENCH_GEMINI_API_KEY' => %i[gemini api_key],
+        'GEMINI_API_KEY' => %i[gemini api_key],
+        'SKILL_BENCH_GEMINI_LOCATION' => %i[gemini location],
+        'GEMINI_LOCATION' => %i[gemini location],
+        'SKILL_BENCH_GEMINI_PROJECT_ID' => %i[gemini project_id],
+        'GEMINI_PROJECT_ID' => %i[gemini project_id],
+        'SKILL_BENCH_GEMINI_MODEL' => %i[gemini model],
+        'GEMINI_MODEL' => %i[gemini model],
+        'SKILL_BENCH_OLLAMA_BASE_URL' => %i[ollama base_url],
+        'OLLAMA_BASE_URL' => %i[ollama base_url],
+        'SKILL_BENCH_OLLAMA_MODEL' => %i[ollama model],
+        'OLLAMA_MODEL' => %i[ollama model],
+        'SKILL_BENCH_AZURE_OPENAI_API_KEY' => %i[azure api_key],
+        'AZURE_OPENAI_API_KEY' => %i[azure api_key],
+        'SKILL_BENCH_AZURE_OPENAI_ENDPOINT' => %i[azure endpoint],
+        'AZURE_OPENAI_ENDPOINT' => %i[azure endpoint],
+        'SKILL_BENCH_AZURE_OPENAI_API_VERSION' => %i[azure api_version],
+        'AZURE_OPENAI_API_VERSION' => %i[azure api_version],
+        'SKILL_BENCH_AZURE_OPENAI_MODEL' => %i[azure model],
+        'AZURE_OPENAI_MODEL' => %i[azure model],
+        'SKILL_BENCH_ANTHROPIC_MODEL' => %i[anthropic model],
+        'ANTHROPIC_MODEL' => %i[anthropic model],
+        'SKILL_BENCH_GROQ_API_KEY' => %i[groq api_key],
+        'SKILL_BENCH_GROQ_MODEL' => %i[groq model],
+        'SKILL_BENCH_DEEPSEEK_API_KEY' => %i[deepseek api_key],
+        'SKILL_BENCH_DEEPSEEK_MODEL' => %i[deepseek model],
+        'SKILL_BENCH_OPENCODE_API_KEY' => %i[opencode api_key],
+        'OPENCODE_API_KEY' => %i[opencode api_key],
+        'SKILL_BENCH_OPENCODE_BASE_URL' => %i[opencode base_url],
+        'OPENCODE_BASE_URL' => %i[opencode base_url],
+        'SKILL_BENCH_OPENCODE_MODEL' => %i[opencode model],
+        'OPENCODE_MODEL' => %i[opencode model],
+        'SKILL_BENCH_OPENROUTER_API_KEY' => %i[openrouter api_key],
+        'SKILL_BENCH_OPENROUTER_MODEL' => %i[openrouter model]
+      }.freeze
+
+      # Returns provider overrides from the given environment.
+      #
+      # @param env [Hash] environment-like object keyed by variable name
+      # @return [Hash] result envelope with provider configuration overrides
+      def self.call(env: ENV)
+        new(env:).call
+      end
+
+      # Initializes the override builder.
+      #
+      # @param env [Hash] environment-like object keyed by variable name
+      # @return [EnvOverrides] an override builder instance
+      def initialize(env:)
+        @env = env
+      end
+
+      # Returns provider overrides from configured environment variables.
+      #
+      # @return [Hash] result envelope with provider configuration overrides
+      def call
+        { success: true, response: { overrides: provider_overrides } }
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'EnvOverrides Error')
+        { success: false, response: { error: { message: e.message } } }
+      end
+
+      # Mutable accumulator for provider override hashes.
+      class ProviderOverrides
+        # Assigns one provider override.
+        #
+        # @param provider [Symbol] provider name
+        # @param setting [Symbol] provider setting
+        # @param value [Object] override value
+        # @return [Object] assigned value
+        def assign(provider, setting, value)
+          provider_overrides(provider)[setting] = value
+        end
+
+        # Returns accumulated overrides as a hash.
+        #
+        # @return [Hash] provider configuration overrides
+        def to_h
+          @to_h ||= {}
+        end
+
+        private
+
+        def provider_overrides(provider)
+          to_h.fetch(provider) { to_h[provider] = {} }
+        end
+      end
+
+      private
+
+      def provider_overrides
+        ENV_TO_PROVIDER_SETTINGS.each_with_object(ProviderOverrides.new) do |(env_key, (provider, setting)), overrides|
+          value = @env.fetch(env_key, nil)
+          overrides.assign(provider, setting, value) if value
+        end.to_h
+      end
+    end
+  end
+end

data/lib/skill_bench/config/facade_readers.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module SkillBench
+  class Config
+    # Reader methods exposed by the Config facade.
+    module FacadeReaders
+      # Returns the current LLM provider.
+      #
+      # @return [Symbol, nil] current provider
+      def current_llm_provider
+        store.current_llm_provider
+      end
+
+      # Returns maximum command execution time.
+      #
+      # @return [Integer, nil] maximum execution time in seconds
+      def max_execution_time
+        store.max_execution_time
+      end
+
+      # Returns allowed command list.
+      #
+      # @return [Array<String>, nil] allowed commands
+      def allowed_commands
+        store.allowed_commands
+      end
+
+      # Returns provider configuration.
+      #
+      # @return [Hash] provider configuration by provider name
+      def llm_providers_config
+        store.llm_providers_config
+      end
+
+      # Returns the API key for the current LLM provider.
+      #
+      # @return [String, nil] API key for the current provider
+      def api_key
+        store.api_key
+      end
+
+      # Returns the model for the current LLM provider.
+      #
+      # @return [String, nil] model name for the current provider
+      def model
+        store.model
+      end
+
+      # Returns the base URL for the current LLM provider.
+      #
+      # @return [String, nil] base URL for the current provider
+      def base_url
+        store.base_url
+      end
+
+      # Returns configuration for a specific provider.
+      #
+      # @param provider [Symbol] provider name
+      # @return [Hash] configuration for the provider
+      def for_provider(provider)
+        store.for_provider(provider)
+      end
+    end
+  end
+end

data/lib/skill_bench/config/facade_writers.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module SkillBench
+  class Config
+    # Writer methods exposed by the Config facade.
+    module FacadeWriters
+      # Public writer method names mapped to provider setting keys.
+      PROVIDER_SETTINGS = {
+        api_key: :api_key,
+        model: :model,
+        location: :location,
+        project_id: :project_id,
+        base_url: :base_url,
+        endpoint: :endpoint,
+        api_version: :api_version
+      }.freeze
+
+      # Dynamically sets a specific provider's API key.
+      #
+      # @param provider [String, Symbol] provider name
+      # @param key [String, nil] provider API key
+      # @return [String, nil] assigned API key
+      def set_provider_api_key(provider, key)
+        set_provider_setting(provider, PROVIDER_SETTINGS.fetch(:api_key), key)
+      end
+
+      # Dynamically sets a specific provider's model.
+      #
+      # @param provider [String, Symbol] provider name
+      # @param model_name [String] provider model name
+      # @return [String] assigned model name
+      def set_provider_model(provider, model_name)
+        set_provider_setting(provider, PROVIDER_SETTINGS.fetch(:model), model_name)
+      end
+
+      # Dynamically sets a specific provider's location.
+      #
+      # @param provider [String, Symbol] provider name
+      # @param location_name [String] provider location
+      # @return [String] assigned location
+      def set_provider_location(provider, location_name)
+        set_provider_setting(provider, PROVIDER_SETTINGS.fetch(:location), location_name)
+      end
+
+      # Dynamically sets a specific provider's project ID.
+      #
+      # @param provider [String, Symbol] provider name
+      # @param project_id_value [String, nil] provider project ID
+      # @return [String, nil] assigned project ID
+      def set_provider_project_id(provider, project_id_value)
+        set_provider_setting(provider, PROVIDER_SETTINGS.fetch(:project_id), project_id_value)
+      end
+
+      # Dynamically sets a specific provider's base URL.
+      #
+      # @param provider [String, Symbol] provider name
+      # @param base_url_value [String, nil] provider base URL
+      # @return [String, nil] assigned base URL
+      def set_provider_base_url(provider, base_url_value)
+        set_provider_setting(provider, PROVIDER_SETTINGS.fetch(:base_url), base_url_value)
+      end
+
+      # Dynamically sets a specific provider's endpoint (Azure OpenAI).
+      #
+      # @param provider [String, Symbol] provider name
+      # @param endpoint_value [String, nil] provider endpoint URL
+      # @return [String, nil] assigned endpoint
+      def set_provider_endpoint(provider, endpoint_value)
+        set_provider_setting(provider, PROVIDER_SETTINGS.fetch(:endpoint), endpoint_value)
+      end
+
+      # Dynamically sets a specific provider's API version (Azure OpenAI).
+      #
+      # @param provider [String, Symbol] provider name
+      # @param version_value [String, nil] provider API version
+      # @return [String, nil] assigned API version
+      def set_provider_api_version(provider, version_value)
+        set_provider_setting(provider, PROVIDER_SETTINGS.fetch(:api_version), version_value)
+      end
+
+      # Sets the current LLM provider.
+      #
+      # @param value [String, Symbol] provider name
+      # @return [Symbol, nil] assigned provider name
+      def current_llm_provider=(value)
+        store.assign_current_llm_provider(value)
+      end
+
+      # Sets the maximum command execution time.
+      #
+      # @param value [Integer] maximum execution time in seconds
+      # @return [Integer] assigned maximum execution time
+      def max_execution_time=(value)
+        store.assign_max_execution_time(value)
+      end
+
+      # Sets the allowed command list.
+      #
+      # @param value [Array<String>, nil] allowed command list
+      # @return [Array<String>, nil] assigned allowed commands
+      def allowed_commands=(value)
+        store.assign_allowed_commands(value)
+      end
+
+      # Replaces provider configuration.
+      #
+      # @param value [Hash] provider configuration
+      # @return [Hash] assigned provider configuration
+      def llm_providers_config=(value)
+        store.replace_provider_config(value)
+      end
+
+      private
+
+      def set_provider_setting(provider, setting, value)
+        store.set_provider_setting(provider, setting, value)
+      end
+    end
+  end
+end

data/lib/skill_bench/config/json_loader.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module SkillBench
+  class Config
+    # Loads and normalizes evaluator JSON configuration files.
+    class JsonLoader
+      # Loads a JSON config file into a normalized hash.
+      #
+      # @param path [Pathname] path to the JSON configuration file
+      # @return [Hash] result envelope with normalized configuration values
+      def self.call(path)
+        new(path).call
+      end
+
+      # Initializes the loader.
+      #
+      # @param path [Pathname] path to the JSON configuration file
+      # @return [JsonLoader] a loader instance
+      def initialize(path)
+        @path = path
+      end
+
+      # Loads a JSON config file into a normalized hash.
+      #
+      # @return [Hash] result envelope with normalized configuration values
+      def call
+        data = JSON.parse(File.read(@path), symbolize_names: true)
+        return warn_invalid_config unless data.is_a?(Hash)
+
+        success(data.slice(:current_llm_provider, :max_execution_time, :allowed_commands)
+                    .compact
+                    .merge(providers: normalized_providers(data[:providers])))
+      rescue JSON::ParserError => e
+        log_parse_error(e)
+        failure('Failed to parse config file')
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'JsonLoader Error')
+        failure(e.message)
+      end
+
+      private
+
+      def warn_invalid_config
+        warn "Warning: Config file at #{@path} is not a valid JSON hash. Skipping."
+        failure('Config file is not a valid JSON hash')
+      end
+
+      def normalized_providers(providers_data)
+        providers_data ||= {}
+        return warn_invalid_providers unless providers_data.is_a?(Hash)
+
+        providers_data.each_with_object({}) do |(provider, config), providers|
+          if config.is_a?(Hash)
+            providers[provider] = config
+          else
+            warn "Warning: provider '#{provider}' in config file at #{@path} is not a valid hash. Skipping."
+          end
+        end
+      end
+
+      def warn_invalid_providers
+        warn "Warning: 'providers' section in config file at #{@path} is not a valid hash. Skipping provider merge."
+        {}
+      end
+
+      def log_parse_error(error)
+        warn "Warning: Failed to parse config file at #{@path}. It might be malformed or empty."
+        warn error.message
+        backtrace = Array(error.backtrace).first(5)
+        warn backtrace.join("\n") unless backtrace.empty?
+      end
+
+      def success(config)
+        { success: true, response: { config: config } }
+      end
+
+      def failure(message)
+        { success: false, response: { error: { message: message } } }
+      end
+    end
+  end
+end

data/lib/skill_bench/config/store.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+module SkillBench
+  class Config
+    # Holds mutable evaluator configuration state behind the Config facade.
+    class Store
+      # Returns the current provider name.
+      #
+      # @return [Symbol, nil] current provider
+      attr_accessor :current_llm_provider
+
+      # Returns the maximum command execution time.
+      #
+      # @return [Integer, nil] maximum execution time in seconds
+      attr_reader :max_execution_time
+
+      # Returns the allowed command list.
+      #
+      # @return [Array<String>, nil] allowed commands
+      attr_accessor :allowed_commands
+
+      # Returns provider configuration.
+      #
+      # @return [Hash, nil] provider configuration by provider name
+      attr_accessor :llm_providers_config
+
+      # Initializes a new configuration store with empty provider settings.
+      def initialize
+        @llm_providers_config = {}
+      end
+
+      # Returns the API key for the current provider.
+      #
+      # @return [String, nil] configured API key
+      def api_key
+        llm_providers_config.dig(current_llm_provider, :api_key)
+      end
+
+      # Returns the model for the current provider.
+      #
+      # @return [String, nil] configured model name
+      def model
+        llm_providers_config.dig(current_llm_provider, :model)
+      end
+
+      # Returns the base URL for the current provider.
+      #
+      # @return [String, nil] configured base URL
+      def base_url
+        llm_providers_config.dig(current_llm_provider, :base_url)
+      end
+
+      # Returns configuration for a specific provider.
+      #
+      # @param provider [Symbol] provider name
+      # @return [Hash] configuration for the provider
+      def for_provider(provider)
+        llm_providers_config[provider.to_sym] || {}
+      end
+
+      # Applies provider-specific configuration values.
+      #
+      # @param providers [Hash] provider configuration by provider name
+      # @return [Hash] provider configuration
+      def apply_provider_config(providers)
+        providers.each do |provider, config|
+          provider_config(provider).merge!(config)
+        end
+      end
+
+      # Sets one provider-specific configuration value.
+      #
+      # @param provider [String, Symbol] provider name
+      # @param setting [Symbol] provider setting name
+      # @param value [Object] provider setting value
+      # @return [Object] assigned value
+      def set_provider_setting(provider, setting, value)
+        provider_config(provider)[setting] = value
+      end
+
+      # Sets the current provider.
+      #
+      # @param value [String, Symbol] provider name
+      # @return [Symbol, nil] assigned provider
+      def assign_current_llm_provider(value)
+        stripped = value.to_s.strip
+        @current_llm_provider = stripped.empty? ? nil : stripped.to_sym
+        @current_llm_provider
+      end
+
+      # Sets maximum command execution time.
+      #
+      # @param value [Integer] maximum execution time in seconds
+      # @return [Integer] assigned maximum execution time
+      def assign_max_execution_time(value)
+        @max_execution_time = value
+      end
+
+      # Sets allowed commands.
+      #
+      # @param value [Array<String>, nil] allowed command list
+      # @return [Array<String>, nil] assigned allowed commands
+      def assign_allowed_commands(value)
+        @allowed_commands = value
+      end
+
+      # Sets provider configuration.
+      #
+      # @param value [Hash] provider configuration
+      # @return [Hash] assigned provider configuration
+      def replace_provider_config(value)
+        @llm_providers_config = value
+      end
+
+      # Sets API key for a specific provider.
+      #
+      # @param provider [Symbol] provider name
+      # @param api_key [String] API key value
+      # @return [String] assigned API key
+      def set_provider_api_key(provider, api_key)
+        provider_config(provider)[:api_key] = api_key
+      end
+
+      # Sets model for a specific provider.
+      #
+      # @param provider [Symbol] provider name
+      # @param model [String] model name
+      # @return [String] assigned model
+      def set_provider_model(provider, model)
+        provider_config(provider)[:model] = model
+      end
+
+      # Sets endpoint for a specific provider.
+      #
+      # @param provider [Symbol] provider name
+      # @param endpoint [String] endpoint URL
+      # @return [String] assigned endpoint
+      def set_provider_endpoint(provider, endpoint)
+        provider_config(provider)[:endpoint] = endpoint
+      end
+
+      # Sets location for a specific provider.
+      #
+      # @param provider [Symbol] provider name
+      # @param location [String] location
+      # @return [String] assigned location
+      def set_provider_location(provider, location)
+        provider_config(provider)[:location] = location
+      end
+
+      # Sets project_id for a specific provider.
+      #
+      # @param provider [Symbol] provider name
+      # @param project_id [String] project ID
+      # @return [String] assigned project_id
+      def set_provider_project_id(provider, project_id)
+        provider_config(provider)[:project_id] = project_id
+      end
+
+      # Sets base_url for a specific provider.
+      #
+      # @param provider [Symbol] provider name
+      # @param base_url [String] base URL
+      # @return [String] assigned base_url
+      def set_provider_base_url(provider, base_url)
+        provider_config(provider)[:base_url] = base_url
+      end
+
+      private
+
+      def provider_config(provider)
+        @llm_providers_config ||= {}
+        @llm_providers_config[provider.to_sym] ||= {}
+      end
+    end
+  end
+end