ruby-skill-bench 1.1.0 → 1.3.0

data/lib/skill_bench/clients/base_url_validator.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'uri'
+
+module SkillBench
+  module Clients
+    # Validates a provider `base_url` before it is used to build an HTTP
+    # connection that may carry an API key / bearer token.
+    #
+    # Security rationale: `base_url` is taken verbatim from config/env input and
+    # the authenticated request attaches a credential to whatever host it names.
+    # Left unchecked this is an SSRF surface, and an `http://` URL would transmit
+    # the credential in cleartext. This service enforces:
+    #
+    # - the URL must be an absolute `http`/`https` URL with a host (empty/relative
+    #   /garbage values are rejected);
+    # - when a credential will be attached, non-loopback hosts MUST use `https`;
+    #   loopback hosts (`localhost`, `127.0.0.1`, `::1`) MAY use `http` — the
+    #   legitimate self-hosted/Ollama case — and an explicit opt-in
+    #   (`allow_insecure_base_url`) can permit cleartext for non-loopback hosts.
+    #
+    # A blank (`nil`/empty) `base_url` is allowed so providers may supply their
+    # own (https) default downstream. Error messages describe only the transport
+    # and never include the credential.
+    class BaseUrlValidator
+      # Hosts permitted to use cleartext `http` even with a credential attached.
+      LOOPBACK_HOSTS = %w[localhost 127.0.0.1 ::1].freeze
+
+      # Raised when a base URL is structurally invalid or would leak a credential
+      # over cleartext transport. The message never contains the credential.
+      class InvalidBaseURLError < StandardError; end
+
+      # Validates a base URL and returns it unchanged when valid.
+      #
+      # @param base_url [String, nil] the URL to validate; blank values are
+      #   returned as-is so a provider default can be applied later.
+      # @param has_credential [Boolean] whether a credential (api key/bearer
+      #   token) will be attached to requests sent to this URL.
+      # @param allow_insecure [Boolean] explicit opt-in that permits cleartext
+      #   `http` to a non-loopback host even when a credential is attached.
+      # @raise [InvalidBaseURLError] when the URL is invalid or insecure.
+      # @return [String, nil] the validated URL (blank input returned unchanged).
+      def self.call(base_url:, has_credential: false, allow_insecure: false)
+        new(base_url, has_credential, allow_insecure).call
+      end
+
+      # @param base_url [String, nil] the URL to validate.
+      # @param has_credential [Boolean] whether a credential will be attached.
+      # @param allow_insecure [Boolean] opt-in permitting cleartext non-loopback.
+      def initialize(base_url, has_credential, allow_insecure)
+        @base_url = base_url
+        @has_credential = has_credential
+        @allow_insecure = allow_insecure
+      end
+
+      # Runs the validation.
+      #
+      # @raise [InvalidBaseURLError] when the URL is invalid or insecure.
+      # @return [String, nil] the validated URL.
+      def call
+        return @base_url if blank?(@base_url)
+
+        validate_absolute_http_url!
+        validate_secure_transport!
+        @base_url
+      end
+
+      private
+
+      def blank?(value)
+        value.to_s.strip.empty?
+      end
+
+      def uri
+        @uri ||= URI.parse(@base_url.to_s)
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      def validate_absolute_http_url!
+        return if uri.is_a?(URI::HTTP) && !blank?(uri.hostname)
+
+        raise InvalidBaseURLError,
+              "Invalid provider base_url #{@base_url.inspect}: " \
+              'must be an absolute http(s) URL with a host.'
+      end
+
+      def validate_secure_transport!
+        return unless @has_credential
+        return if uri.scheme == 'https'
+        return if loopback?
+        return if @allow_insecure
+
+        raise InvalidBaseURLError,
+              'Insecure provider base_url: refusing to send a credential over cleartext http ' \
+              "to non-loopback host #{uri.hostname.inspect}. Use https, target a loopback host, " \
+              'or set allow_insecure_base_url: true to override.'
+      end
+
+      def loopback?
+        LOOPBACK_HOSTS.include?(uri.hostname)
+      end
+    end
+  end
+end

data/lib/skill_bench/clients/provider_config.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative '../config'
+require_relative 'base_url_validator'
 
 module SkillBench
   module Clients
@@ -13,6 +14,8 @@ module SkillBench
         new(provider, options).call
       end
 
+      # @param provider [Symbol, String] provider identifier, coerced to a Symbol (e.g., :openai, :ollama)
+      # @param options [Hash] override options that take precedence over the loaded provider config
       def initialize(provider, options)
         @provider = provider.to_sym
         @options = options
@@ -21,8 +24,21 @@ module SkillBench
 
       # Loads and returns standardized provider configuration.
       #
+      # The resolved transport URLs (`base_url` and, for Azure, `endpoint`) are
+      # validated before being returned: they must be absolute http(s) URLs, and
+      # a credential is never sent over cleartext http to a non-loopback host.
+      #
+      # @raise [BaseUrlValidator::InvalidBaseURLError] when a transport URL is
+      #   structurally invalid or would leak the credential over cleartext http.
       # @return [Hash] Standardized configuration with api_key, model, base_url, etc.
       def call
+        validate_transport_urls!
+        standardized_config
+      end
+
+      private
+
+      def standardized_config
         {
           api_key: fetch_config(:api_key),
           model: fetch_config(:model),
@@ -37,10 +53,33 @@ module SkillBench
         }
       end
 
-      private
+      # Validates every transport URL that could carry the credential. Both
+      # `base_url` and Azure's `endpoint` are user-supplied URLs that the
+      # authenticated request targets, so both are checked with one helper.
+      #
+      # @raise [BaseUrlValidator::InvalidBaseURLError] on an invalid/insecure URL.
+      # @return [void]
+      def validate_transport_urls!
+        has_credential = !fetch_config(:api_key).to_s.empty?
+        allow_insecure = truthy?(fetch_config(:allow_insecure_base_url))
+
+        [fetch_config(:base_url), fetch_config(:endpoint)].each do |url|
+          BaseUrlValidator.call(base_url: url, has_credential: has_credential, allow_insecure: allow_insecure)
+        end
+      end
 
+      def truthy?(value)
+        value == true || value.to_s.strip.casecmp?('true')
+      end
+
+      # Prefer an option when the key is present — including an explicit `nil`,
+      # which means "unset" and must not fall through to global Config (a prior
+      # `||` treated nil as missing and leaked stored API keys into clients).
       def fetch_config(key)
-        @options[key] || @config[key]
+        return @options[key] if @options.key?(key)
+        return @options[key.to_s] if @options.key?(key.to_s)
+
+        @config[key] || @config[key.to_s]
       end
     end
   end

data/lib/skill_bench/clients/provider_schemas.rb

@@ -39,6 +39,10 @@ module SkillBench
           api_key: nil,
           model: 'deepseek-chat'
         }.freeze,
+        mistral: {
+          api_key: nil,
+          model: 'mistral-large-latest'
+        }.freeze,
         opencode: {
           api_key: nil,
           model: 'opencode-model',

data/lib/skill_bench/clients/providers/mistral.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require_relative '../base_client'
+require_relative '../provider_registry'
+
+module SkillBench
+  module Clients
+    module Providers
+      # Mistral (la Plateforme) LLM client.
+      # Uses Mistral's OpenAI-compatible chat completions API with bearer-token auth.
+      #
+      # NOTE: AWS Bedrock access to Mistral models (which requires SigV4 request
+      # signing rather than a static bearer token) is intentionally not handled
+      # here and is left as a follow-up.
+      class Mistral < BaseClient
+        SkillBench::Clients::ProviderRegistry.register(:mistral, self)
+
+        # Returns the provider identifier.
+        #
+        # @return [Symbol]
+        def provider_name
+          :mistral
+        end
+
+        protected
+
+        # Returns the base URL for the Mistral API.
+        #
+        # The Mistral API base is https://api.mistral.ai/v1; the version segment
+        # lives in {#request_path} so Faraday does not drop it (an absolute
+        # request path replaces any path component of the connection base URL).
+        #
+        # @return [String]
+        def base_url
+          @base_url_config || 'https://api.mistral.ai'
+        end
+
+        # Returns the request path for chat completions.
+        #
+        # @return [String]
+        def request_path
+          @request_path_config || '/v1/chat/completions'
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/commands/init.rb

@@ -24,10 +24,15 @@ module SkillBench
 
       # Generates configuration hash for a specific provider.
       #
+      # The built-in `:mock` provider needs no credentials, so it produces a
+      # minimal offline config without a nested `config:` block.
+      #
       # @param provider [Symbol] LLM provider name
       # @return [Hash] Single-provider configuration
       # @raise [ArgumentError] if provider is not registered
       def self.config_for_provider(provider)
+        return { provider: :mock, max_execution_time: 30 } if provider == :mock
+
         {
           provider: provider,
           max_execution_time: 30,

data/lib/skill_bench/commands/skill_new.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require 'fileutils'
-require_relative '../rails/skill_templates'
 
 module SkillBench
   module Commands
@@ -107,6 +106,9 @@ module SkillBench
         file_name = RAILS_TEMPLATES[template]
         raise ArgumentError, "Invalid template: #{template}. Use one of: #{RAILS_TEMPLATES.keys.join(', ')}." unless file_name
 
+        # Lazily load the scaffold generator so a normal `skill-bench run` does
+        # not pull it (and its dependencies) in at boot.
+        require_relative '../rails/skill_templates'
         content = Rails::SkillTemplates.public_send(template.to_sym, name)
         File.write(File.join(path, file_name), content)
       end

data/lib/skill_bench/config/applier.rb

@@ -41,6 +41,8 @@ module SkillBench
         assign_current_provider
         @store.assign_max_execution_time(@data[:max_execution_time]) if @data.key?(:max_execution_time)
         @store.assign_allowed_commands(@data[:allowed_commands]) if @data.key?(:allowed_commands)
+        @store.assign_allow_host_execution(@data[:allow_host_execution]) if @data.key?(:allow_host_execution)
+        @store.assign_command_argument_constraints(@data[:command_argument_constraints]) if @data.key?(:command_argument_constraints)
         @store.skill_sources = @data[:skill_sources] if @data.key?(:skill_sources)
       end
 

data/lib/skill_bench/config/defaults.rb

@@ -19,6 +19,8 @@ module SkillBench
           current_llm_provider: :openai,
           max_execution_time: 30,
           allowed_commands: nil,
+          allow_host_execution: false,
+          command_argument_constraints: {},
           skill_sources: {},
           llm_providers_config: {
             openai: { api_key: nil, model: 'gpt-4o' },

data/lib/skill_bench/config/facade_readers.rb

@@ -25,6 +25,13 @@ module SkillBench
         store.allowed_commands
       end
 
+      # Returns whether un-isolated host command execution is permitted.
+      #
+      # @return [Boolean, nil] true when host execution is explicitly allowed
+      def allow_host_execution
+        store.allow_host_execution
+      end
+
       # Returns provider configuration.
       #
       # @return [Hash] provider configuration by provider name

data/lib/skill_bench/config/facade_writers.rb

@@ -102,6 +102,23 @@ module SkillBench
         store.assign_allowed_commands(value)
       end
 
+      # Sets whether un-isolated host command execution is permitted.
+      #
+      # @param value [Boolean] true to permit un-isolated host execution
+      # @return [Boolean] assigned host execution flag
+      def allow_host_execution=(value)
+        store.assign_allow_host_execution(value)
+      end
+
+      # Sets the optional per-command argument constraints.
+      #
+      # @param value [Hash, nil] base command => disallowed argument
+      #   substrings/flags
+      # @return [Hash, nil] assigned constraints
+      def command_argument_constraints=(value)
+        store.assign_command_argument_constraints(value)
+      end
+
       # Replaces provider configuration.
       #
       # @param value [Hash] provider configuration

data/lib/skill_bench/config/json_loader.rb

@@ -29,7 +29,7 @@ module SkillBench
         data = JSON.parse(File.read(@path), symbolize_names: true)
         return warn_invalid_config unless data.is_a?(Hash)
 
-        success_data = data.slice(:current_llm_provider, :max_execution_time, :allowed_commands, :skill_sources).compact
+        success_data = data.slice(:current_llm_provider, :max_execution_time, :allowed_commands, :allow_host_execution, :command_argument_constraints, :skill_sources).compact
         success_data[:current_llm_provider] ||= data[:provider] if data.key?(:provider)
         success(success_data.merge(providers: normalized_providers(data[:providers])))
       rescue JSON::ParserError => e

data/lib/skill_bench/config/store.rb

@@ -19,6 +19,18 @@ module SkillBench
       # @return [Array<String>, nil] allowed commands
       attr_accessor :allowed_commands
 
+      # Returns whether running commands directly on the host is permitted
+      # when no real sandbox isolation (container) is active.
+      #
+      # @return [Boolean, nil] true when host execution is explicitly allowed
+      attr_reader :allow_host_execution
+
+      # Returns the optional per-command argument constraints.
+      #
+      # @return [Hash, nil] base command => disallowed argument
+      #   substrings/flags, or nil when unconfigured
+      attr_reader :command_argument_constraints
+
       # Returns provider configuration.
       #
       # @return [Hash, nil] provider configuration by provider name
@@ -109,6 +121,23 @@ module SkillBench
         @allowed_commands = value
       end
 
+      # Sets whether host command execution is permitted without isolation.
+      #
+      # @param value [Boolean] true to permit un-isolated host execution
+      # @return [Boolean] assigned host execution flag
+      def assign_allow_host_execution(value)
+        @allow_host_execution = value
+      end
+
+      # Sets the optional per-command argument constraints.
+      #
+      # @param value [Hash, nil] base command => disallowed argument
+      #   substrings/flags
+      # @return [Hash, nil] assigned constraints
+      def assign_command_argument_constraints(value)
+        @command_argument_constraints = value
+      end
+
       # Sets provider configuration.
       #
       # @param value [Hash] provider configuration

data/lib/skill_bench/config.rb

@@ -95,6 +95,24 @@ module SkillBench
         store.allowed_commands
       end
 
+      # Returns whether commands may run directly on the host when no sandbox
+      # isolation (container) is active. Defaults to false (fail closed).
+      #
+      # @return [Boolean] true when un-isolated host execution is explicitly enabled
+      def allow_host_execution
+        store.allow_host_execution || false
+      end
+
+      # Returns the optional per-command argument constraints.
+      #
+      # When unconfigured, returns an empty Hash meaning no argument constraints
+      # apply (the allowlist remains the only command-authorization control).
+      #
+      # @return [Hash] base command => disallowed argument substrings/flags
+      def command_argument_constraints
+        store.command_argument_constraints || {}
+      end
+
       # Returns max execution time from configuration.
       #
       # @return [Integer] Maximum execution time in seconds

data/lib/skill_bench/constants.rb

@@ -30,6 +30,33 @@ module SkillBench
     # Sandbox Configuration
     module Sandbox
       DOCKER_IMAGE_NAME = 'evaluator-sandbox'
+
+      # Relative path of the Docker build context as packaged in the gem.
+      DOCKER_CONTEXT_RELATIVE = 'lib/skill_bench/execution/docker'
+
+      # Absolute path to the Docker build context shipped next to the library code.
+      # Resolved from this file so gem installs and git checkouts agree.
+      #
+      # @return [String] absolute path to the docker/ directory
+      def self.docker_context_path
+        File.expand_path('execution/docker', __dir__)
+      end
+
+      # Versioned image reference used for build and run.
+      #
+      # @return [String] e.g. "evaluator-sandbox:1.2.0"
+      def self.image_ref
+        require_relative 'version' unless defined?(SkillBench::VERSION)
+
+        "#{DOCKER_IMAGE_NAME}:#{SkillBench::VERSION}"
+      end
+
+      # Floating latest tag applied alongside the versioned tag on build.
+      #
+      # @return [String] e.g. "evaluator-sandbox:latest"
+      def self.latest_image_ref
+        "#{DOCKER_IMAGE_NAME}:latest"
+      end
     end
 
     # Tool Execution Configuration

data/lib/skill_bench/evaluation/runner.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'parallel'
+
 module SkillBench
   module Evaluation
     # Orchestrates the evaluation pipeline.
@@ -39,10 +41,8 @@ module SkillBench
       #
       # @return [Hash] Service response with report or error.
       def call
-        baseline_judge = judge_run(baseline_output, nil)
+        baseline_judge, context_judge = run_judges_concurrently
         return baseline_judge unless baseline_judge[:success]
-
-        context_judge = judge_run(context_output, skill_context)
         return context_judge unless context_judge[:success]
 
         compute_deltas(baseline_judge, context_judge)
@@ -55,6 +55,23 @@ module SkillBench
 
       attr_reader :task, :criteria, :skill_context, :baseline_output, :context_output, :judge_params
 
+      # Judges the baseline and context outputs concurrently.
+      #
+      # The two runs are independent blind evaluations that share no mutable
+      # state, so they execute on separate threads (the LLM round-trip is
+      # I/O-bound and releases the GIL). +Parallel.map+ preserves input order,
+      # so the baseline result is always first and the context result second;
+      # callers still apply the sequential failure precedence afterwards.
+      #
+      # @return [Array(Hash, Hash)] Baseline and context judge results, in order.
+      def run_judges_concurrently
+        runs = [
+          -> { judge_run(baseline_output, nil) },
+          -> { judge_run(context_output, skill_context) }
+        ]
+        Parallel.map(runs, in_threads: runs.size, &:call)
+      end
+
       def judge_run(output, context)
         prompt_result = Judge::Prompt.call(
           task: task,

data/lib/skill_bench/execution/context_hydrator.rb

@@ -12,6 +12,11 @@ module SkillBench
       # Error message returned when context hydration fails.
       HYDRATION_FAILED = 'Failed to hydrate context from source path'
 
+      # Immutable record pairing a context file's path with the content and byte
+      # size captured during a single filesystem pass, so the total-size check and
+      # the XML build can reuse them without a second `stat` or `read`.
+      ContextFile = Struct.new(:path, :content, :bytesize)
+
       # Loads and formats source context files.
       #
       # @param params [Hash] The configuration for context hydration.
@@ -43,7 +48,7 @@ module SkillBench
         full_path = @base_path.join(@source_path).expand_path
         base_expanded = @base_path.expand_path
 
-        return missing_path_result unless full_path.to_path.start_with?(base_expanded.to_path)
+        return missing_path_result unless within_base?(full_path, base_expanded)
         return missing_path_result unless full_path.exist? && full_path.directory?
 
         context_files = collect_context_files(full_path)
@@ -59,19 +64,56 @@ module SkillBench
 
       private
 
+      # Determines whether the resolved path is contained within the base directory.
+      # Uses a separator-aware boundary so a sibling directory whose name merely shares
+      # the base directory's prefix (e.g. base `/tmp/foo` vs `/tmp/foo-evil`) is rejected.
+      #
+      # @param full_path [Pathname] The expanded source path to validate.
+      # @param base_expanded [Pathname] The expanded base directory.
+      # @return [Boolean] true when full_path is the base directory or a descendant of it.
+      def within_base?(full_path, base_expanded)
+        full = full_path.to_path
+        base = base_expanded.to_path
+        full == base || full.start_with?(base + File::SEPARATOR)
+      end
+
       def missing_path_result
         { success: false, response: { error: { message: "Source path #{@source_path} does not exist or is not a directory" } } }
       end
 
+      # Collects readable context files in a single filesystem pass. Symlinks are
+      # rejected and oversized files are skipped via a cheap `File.size` pre-check
+      # so a huge file is never read into memory; each surviving file is read
+      # exactly once, capturing its content and byte size for downstream reuse.
+      #
+      # @param full_path [Pathname] The validated, in-base source directory.
+      # @return [Array<ContextFile>] Sorted records of path, content, and byte size.
       def collect_context_files(full_path)
         pattern = full_path.join("*{#{Constants::ContextHydration::TEXT_EXTENSIONS.join(',')}}").to_s
-        Dir.glob(pattern).reject { |f| File.symlink?(f) }
-                         .select { |f| File.size(f) <= Constants::ContextHydration::MAX_FILE_SIZE }
-                         .sort
+        Dir.glob(pattern)
+           .reject { |file_path| File.symlink?(file_path) }
+           .select { |file_path| File.size(file_path) <= Constants::ContextHydration::MAX_FILE_SIZE }
+           .map { |file_path| read_context_file(file_path) }
       end
 
+      # Reads a single in-limit file once, pairing its content with the byte size
+      # derived from that content so no second `stat` is required.
+      #
+      # @param file_path [String] Absolute path to an in-limit context file.
+      # @return [ContextFile] The path, content, and byte size record.
+      def read_context_file(file_path)
+        content = File.read(file_path)
+        ContextFile.new(file_path, content, content.bytesize)
+      end
+
+      # Validates that the combined byte size of the already-read context files
+      # stays within the total-size cap, reusing the sizes captured during
+      # collection instead of re-stat-ing each file.
+      #
+      # @param context_files [Array<ContextFile>] The collected context records.
+      # @return [Boolean] true when the total size is within the cap.
       def validate_total_size?(context_files)
-        total_size = context_files.sum { |f| File.size(f) }
+        total_size = context_files.sum(&:bytesize)
         return true if total_size <= Constants::ContextHydration::MAX_TOTAL_CONTEXT_SIZE
 
         SkillBench::ErrorLogger.log_error(
@@ -81,21 +123,20 @@ module SkillBench
         false
       end
 
-      # Builds the XML structure wrapping the contents of the context files.
+      # Builds the XML structure wrapping the already-read context file contents.
       #
-      # @param context_files [Array<String>] List of absolute paths to context files.
+      # @param context_files [Array<ContextFile>] The collected context records.
       # @return [String] The combined XML representation of the file contents.
       def build_xml(context_files)
         return '' if context_files.empty?
 
         xml = ['<agent_context>']
 
-        context_files.each do |file_path|
-          relative_path = Pathname.new(file_path).relative_path_from(@base_path).to_s
-          content = File.read(file_path)
+        context_files.each do |context_file|
+          relative_path = Pathname.new(context_file.path).relative_path_from(@base_path).to_s
 
           xml << "  <file path=\"#{CGI.escapeHTML(relative_path)}\">"
-          xml << CGI.escapeHTML(content).gsub(/^/, '    ')
+          xml << CGI.escapeHTML(context_file.content).gsub(/^/, '    ')
           xml << '  </file>'
         end
 

data/lib/skill_bench/execution/docker/.dockerignore

@@ -0,0 +1,2 @@
+*
+!Dockerfile

data/lib/skill_bench/execution/docker/Dockerfile

@@ -0,0 +1,24 @@
+# SkillBench evaluator-sandbox image contract (MVP)
+#
+# Purpose: long-lived container for `docker exec` of allowlisted agent commands.
+# Runtime flags (from Sandbox#start_container): --network none, non-root uid/gid,
+#   --security-opt no-new-privileges, --cap-drop ALL (+ CHOUN/DAC_OVERRIDE).
+#
+# Assumptions:
+# - No package installs or gem fetches during an eval (network is disabled at runtime).
+# - Ruby major tracks CI (3.3/3.4). Bookworm image provides a modern baseline.
+# - git is available for tools that shell out to git inside the sandbox mount.
+# - Multi-arch: build on the host arch (arm64 Mac / amd64 CI) via docker build.
+#
+# See docs/docker.md and milestone container-isolation-v1.
+
+FROM ruby:3.4-bookworm
+
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends git \
+  && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /sandbox
+
+# Keep the container alive so repeated `docker exec` sessions can run commands.
+CMD ["sleep", "infinity"]