ruby-skill-bench 0.1.0 → 1.1.0

data/lib/skill_bench/clients/response_error_handler.rb

@@ -23,14 +23,8 @@ module SkillBench
           error_msg += " - #{detail}"
         end
 
-        {
-          success: false,
-          result: error_msg,
-          usage: usage_extractor.call(parsed),
-          response: { error: { message: error_msg } },
-          status: 'error',
-          code: response.status
-        }
+        base_response = ResponseBuilder.api_error(error_message: error_msg, usage: usage_extractor.call(parsed))
+        base_response.merge(code: response.status)
       end
 
       # Creates an error response when the LLM response has no message content.
@@ -41,14 +35,8 @@ module SkillBench
       # @return [Hash] Standardized error response
       def self.missing_message_response(response, parsed, &usage_extractor)
         error_msg = 'LLM response missing message content'
-        {
-          success: false,
-          result: error_msg,
-          usage: usage_extractor.call(parsed),
-          response: { error: { message: error_msg } },
-          status: 'error',
-          code: response.status
-        }
+        base_response = ResponseBuilder.error(message: error_msg)
+        base_response.merge(usage: usage_extractor.call(parsed), code: response.status)
       end
 
       # Handles an exception by logging and returning a standardized error response.
@@ -58,7 +46,7 @@ module SkillBench
       # @return [Hash] Standardized error response
       def self.handle_exception(error, type)
         log_error(error)
-        { success: false, result: "#{type}: #{error.message}", status: 'error' }
+        ResponseBuilder.error(message: "#{type}: #{error.message}")
       end
 
       # Logs an error message and backtrace to Rails.logger or stderr.

data/lib/skill_bench/clients/retry_handler.rb

@@ -2,6 +2,7 @@
 
 require 'faraday'
 require_relative '../error_logger'
+require_relative '../constants'
 
 module SkillBench
   module Clients
@@ -9,10 +10,6 @@ module SkillBench
     # Retries on transient errors (429, 503). Raises permanent errors immediately.
     # Returns the block result on success.
     class RetryHandler
-      RETRYABLE_STATUSES = [429, 503].freeze
-
-      MAX_DELAY = 30 # Maximum delay cap in seconds
-
       # Executes the given block with retry logic.
       #
       # @param max_attempts [Integer] Maximum number of attempts (default: 3).
@@ -21,7 +18,7 @@ module SkillBench
       # @return [Object] The block's return value on success.
       # @raise [Faraday::Error] On non-retryable errors or after exhausting retries.
       # @raise [ArgumentError] if no block is given or max_attempts < 1.
-      def self.call(max_attempts: 3, base_delay: 1, &block)
+      def self.call(max_attempts: Constants::HttpClient::DEFAULT_MAX_RETRIES, base_delay: Constants::HttpClient::DEFAULT_RETRY_DELAY, &block)
         raise ArgumentError, 'RetryHandler requires a block' unless block
         raise ArgumentError, 'max_attempts must be >= 1' if max_attempts < 1
 
@@ -59,11 +56,11 @@ module SkillBench
       private
 
       def retryable?(status, attempt)
-        RETRYABLE_STATUSES.include?(status) && attempt < @max_attempts
+        Constants::HttpClient::RETRYABLE_STATUSES.include?(status) && attempt < @max_attempts
       end
 
       def compute_delay(attempt)
-        [@base_delay * (2**(attempt - 1)), MAX_DELAY].min
+        [@base_delay * (2**(attempt - 1)), Constants::ReactAgent::DEFAULT_MAX_DELAY].min
       end
 
       def extract_status(error)

data/lib/skill_bench/commands/run.rb

@@ -9,11 +9,15 @@ module SkillBench
       # Run an eval with specified skill(s)
       # @param eval_name [String] Name of eval to run (e.g., 'test-eval' or 'evals/test-eval')
       # @param skill_names [Array<String>] Names of skills to use
+      # @param pack [String, nil] Optional pack name for registry-based skill resolution
+      # @param registry_manifest [String, nil] Optional path to registry.json manifest
       # @return [Hash] Result with pass/fail and score
-      def self.run(eval_name:, skill_names:)
+      def self.run(eval_name:, skill_names:, pack: nil, registry_manifest: nil)
         Services::RunnerService.call(
           eval_name: eval_name,
-          skill_names: skill_names
+          skill_names: skill_names,
+          pack: pack,
+          registry_manifest: registry_manifest
         )
       end
     end

data/lib/skill_bench/config/applier.rb

@@ -41,6 +41,7 @@ module SkillBench
         assign_current_provider
         @store.assign_max_execution_time(@data[:max_execution_time]) if @data.key?(:max_execution_time)
         @store.assign_allowed_commands(@data[:allowed_commands]) if @data.key?(:allowed_commands)
+        @store.skill_sources = @data[:skill_sources] if @data.key?(:skill_sources)
       end
 
       def apply_provider_values

data/lib/skill_bench/config/defaults.rb

@@ -19,6 +19,7 @@ module SkillBench
           current_llm_provider: :openai,
           max_execution_time: 30,
           allowed_commands: nil,
+          skill_sources: {},
           llm_providers_config: {
             openai: { api_key: nil, model: 'gpt-4o' },
             anthropic: { api_key: nil, model: 'claude-sonnet-4-20250514' },

data/lib/skill_bench/config/facade_readers.rb

@@ -32,6 +32,13 @@ module SkillBench
         store.llm_providers_config
       end
 
+      # Returns skill sources mapping.
+      #
+      # @return [Hash, nil] skill source name → directory path
+      def skill_sources
+        store.skill_sources
+      end
+
       # Returns the API key for the current LLM provider.
       #
       # @return [String, nil] API key for the current provider

data/lib/skill_bench/config/json_loader.rb

@@ -29,9 +29,9 @@ module SkillBench
         data = JSON.parse(File.read(@path), symbolize_names: true)
         return warn_invalid_config unless data.is_a?(Hash)
 
-        success(data.slice(:current_llm_provider, :max_execution_time, :allowed_commands)
-                    .compact
-                    .merge(providers: normalized_providers(data[:providers])))
+        success_data = data.slice(:current_llm_provider, :max_execution_time, :allowed_commands, :skill_sources).compact
+        success_data[:current_llm_provider] ||= data[:provider] if data.key?(:provider)
+        success(success_data.merge(providers: normalized_providers(data[:providers])))
       rescue JSON::ParserError => e
         log_parse_error(e)
         failure('Failed to parse config file')

data/lib/skill_bench/config/store.rb

@@ -24,6 +24,11 @@ module SkillBench
       # @return [Hash, nil] provider configuration by provider name
       attr_accessor :llm_providers_config
 
+      # Returns skill sources mapping.
+      #
+      # @return [Hash, nil] skill source name → directory path
+      attr_accessor :skill_sources
+
       # Initializes a new configuration store with empty provider settings.
       def initialize
         @llm_providers_config = {}

data/lib/skill_bench/config.rb

@@ -74,7 +74,9 @@ module SkillBench
         @store = Config::Store.new
         apply_defaults
         apply_json_config(home_config_path)
-        apply_json_config(Pathname.new(Dir.pwd).join(CONFIG_FILENAME))
+        local_path = Pathname.new(Dir.pwd).join(CONFIG_FILENAME)
+        is_workspace_file = File.exist?(File.join(Dir.pwd, 'ruby-skill-bench.gemspec'))
+        apply_json_config(local_path) unless defined?(Minitest) && is_workspace_file
         apply_env_overrides
       end
 
@@ -122,6 +124,13 @@ module SkillBench
         store.llm_providers_config || {}
       end
 
+      # Returns skill sources mapping.
+      #
+      # @return [Hash, nil] skill source name → directory path
+      def skill_sources
+        store.skill_sources || {}
+      end
+
       # Returns API key from configuration.
       #
       # @return [String, nil] API key

data/lib/skill_bench/constants.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module SkillBench
+  # Centralized configuration constants for the SkillBench system.
+  # This eliminates magic numbers and provides a single source of truth
+  # for configurable values across the codebase.
+  module Constants
+    # ReAct Agent Configuration
+    module ReactAgent
+      DEFAULT_MAX_ITERATIONS = 25
+      DEFAULT_MAX_DELAY = 30 # Maximum delay cap in seconds for retry logic
+    end
+
+    # HTTP Client Configuration
+    module HttpClient
+      DEFAULT_OPEN_TIMEOUT = 10
+      DEFAULT_TIMEOUT = 120
+      DEFAULT_MAX_RETRIES = 3
+      DEFAULT_RETRY_DELAY = 1
+      RETRYABLE_STATUSES = [429, 503].freeze
+    end
+
+    # Context Hydration Configuration
+    module ContextHydration
+      MAX_FILE_SIZE = 50_000 # Maximum file size in bytes
+      MAX_TOTAL_CONTEXT_SIZE = 1_000_000 # Maximum total context size in bytes (1MB)
+      TEXT_EXTENSIONS = %w[.md .rb .json .yml .yaml .txt].freeze
+    end
+
+    # Sandbox Configuration
+    module Sandbox
+      DOCKER_IMAGE_NAME = 'evaluator-sandbox'
+    end
+
+    # Tool Execution Configuration
+    module Tools
+      DANGEROUS_COMMANDS = %w[
+        bash sh zsh fish dash ksh csh tcsh
+        python python3 python2 ruby perl node
+        php lua tcl wish
+        curl wget nc ncat socat
+        eval exec
+        sudo su doas
+        chmod chown mount umount
+        dd mkfs fdisk parted
+        insmod rmmod modprobe
+        systemctl service
+        passwd useradd userdel groupadd groupdel
+      ].freeze
+    end
+
+    # File Path Configuration
+    module FilePath
+      ALLOWED_PATH_PATTERN = %r{\A[a-zA-Z0-9._\-/]+\z}
+      MAX_PATH_LENGTH = 4096
+    end
+  end
+end

data/lib/skill_bench/delta_report.rb

@@ -49,6 +49,26 @@ module SkillBench
       { success: false, response: { error: { message: e.message } } }
     end
 
+    # Compatibility methods for ComparisonReporter
+
+    # Returns the list of dimensions from the context run.
+    #
+    # @return [Array<Object>] List of objects responding to name and score
+    def dimensions
+      return [] unless context_dimensions
+
+      context_dimensions.map do |name, dim_hash|
+        Struct.new(:name, :score).new(name.to_s, dim_hash[:score] || dim_hash['score'])
+      end
+    end
+
+    # Returns the total context score.
+    #
+    # @return [Numeric, nil]
+    def total
+      context_total
+    end
+
     private
 
     attr_reader :baseline, :context

data/lib/skill_bench/execution/context_hydrator.rb

@@ -2,6 +2,7 @@
 
 require 'pathname'
 require 'cgi'
+require_relative '../constants'
 
 module SkillBench
   module Execution
@@ -10,10 +11,6 @@ module SkillBench
     class ContextHydrator
       # Error message returned when context hydration fails.
       HYDRATION_FAILED = 'Failed to hydrate context from source path'
-      # File extensions considered for context hydration.
-      TEXT_EXTENSIONS = %w[.md .rb .json .yml .yaml .txt].freeze
-      # Maximum file size (in bytes) for files included in context hydration.
-      MAX_FILE_SIZE = 50_000
 
       # Loads and formats source context files.
       #
@@ -50,6 +47,8 @@ module SkillBench
         return missing_path_result unless full_path.exist? && full_path.directory?
 
         context_files = collect_context_files(full_path)
+        return missing_path_result unless validate_total_size?(context_files)
+
         xml_context = build_xml(context_files)
 
         { success: true, response: { context: xml_context } }
@@ -65,12 +64,23 @@ module SkillBench
       end
 
       def collect_context_files(full_path)
-        pattern = full_path.join("*{#{TEXT_EXTENSIONS.join(',')}}").to_s
+        pattern = full_path.join("*{#{Constants::ContextHydration::TEXT_EXTENSIONS.join(',')}}").to_s
         Dir.glob(pattern).reject { |f| File.symlink?(f) }
-                         .select { |f| File.size(f) <= MAX_FILE_SIZE }
+                         .select { |f| File.size(f) <= Constants::ContextHydration::MAX_FILE_SIZE }
                          .sort
       end
 
+      def validate_total_size?(context_files)
+        total_size = context_files.sum { |f| File.size(f) }
+        return true if total_size <= Constants::ContextHydration::MAX_TOTAL_CONTEXT_SIZE
+
+        SkillBench::ErrorLogger.log_error(
+          StandardError.new("Total context size #{total_size} exceeds maximum #{Constants::ContextHydration::MAX_TOTAL_CONTEXT_SIZE}"),
+          'ContextHydrator'
+        )
+        false
+      end
+
       # Builds the XML structure wrapping the contents of the context files.
       #
       # @param context_files [Array<String>] List of absolute paths to context files.

data/lib/skill_bench/execution/sandbox.rb

@@ -3,6 +3,7 @@
 require 'fileutils'
 require 'tmpdir'
 require 'open3'
+require_relative '../constants'
 
 module SkillBench
   module Execution
@@ -143,18 +144,32 @@ module SkillBench
 
       # Starts a Docker container for isolated command execution.
       # Builds the image only if it does not already exist.
+      # Uses hardened security settings for production safety.
       #
       # @raise [RuntimeError] when the Docker image cannot be built or the container fails to start.
       def start_container
-        image_name = 'evaluator-sandbox'
+        image_name = Constants::Sandbox::DOCKER_IMAGE_NAME
         docker_dir = File.expand_path('docker', __dir__)
 
         # Build image (Docker layer cache handles no-op builds)
         raise "Failed to build Docker image #{image_name}" unless system('docker', 'build', '-t', image_name, docker_dir, '--quiet')
 
-        # Start a detached container mounting the sandbox dir to /sandbox
+        # Start a detached container with hardened security settings
+        # --user $(id -u):$(id -g): Runs as non-root user
+        # --security-opt no-new-privileges: Prevents privilege escalation
+        # --cap-drop ALL: Drops all Linux capabilities
+        # --cap-add CHOWN, DAC_OVERRIDE: Adds back minimal capabilities for git operations
+        # --network none: Disables network access for additional isolation
         stdout, stderr, status = Open3.capture3(
-          'docker', 'run', '-d', '--rm', '-v', "#{@path}:/sandbox", image_name
+          'docker', 'run', '-d', '--rm',
+          '--user', "#{Process.uid}:#{Process.gid}",
+          '--security-opt', 'no-new-privileges',
+          '--cap-drop', 'ALL',
+          '--cap-add', 'CHOWN',
+          '--cap-add', 'DAC_OVERRIDE',
+          '--network', 'none',
+          '-v', "#{@path}:/sandbox:rw",
+          image_name
         )
 
         raise "Failed to start Docker container: #{stderr}" unless status.success?

data/lib/skill_bench/execution/source_path_resolver.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'pathname'
+
 module SkillBench
   module Execution
     # Resolves the source skill or workflow path for a given evaluation target.
@@ -8,6 +10,8 @@ module SkillBench
       #
       # @param eval_folder_path [String] Relative path to the eval directory.
       # @param skill_path [String, nil] Optional explicit override for the source directory.
+      # @param skill_sources [Hash] Optional skill source name → directory path mapping for fallback.
+      #   When provided and local resolution does not yield an existing path, each source is checked.
       # @return [String, nil] The resolved source path relative to the evaluator repo root, or nil if unmappable.
       # @example Infer a skill source path (NEW format):
       #   SkillBench::Execution::SourcePathResolver.call(
@@ -19,12 +23,57 @@ module SkillBench
       #     eval_folder_path: 'evals/skills/code-quality/rails-code-review/review-order'
       #   )
       #   # => "skills/code-quality/rails-code-review"
-      def self.call(eval_folder_path:, skill_path: nil)
+      def self.call(eval_folder_path:, skill_path: nil, skill_sources: {})
         return skill_path if skill_path && !skill_path.empty?
 
-        segments = eval_folder_path.to_s.split('/').reject(&:empty?)
+        segments = Pathname.new(eval_folder_path.to_s).each_filename.to_a
+
+        local = resolve_skills_path(segments) || resolve_workflows_path(segments)
+
+        unless local.nil? || skill_sources.empty?
+          skill_name = extract_skill_name(segments)
+          return local unless skill_name
+          return local if skill_exists_at?(local)
+
+          skill_sources.each_value do |source_path|
+            candidate = find_skill_in_source(source_path, skill_name)
+            return candidate if candidate
+          end
+        end
+
+        local
+      end
+
+      # Extracts the skill name from the eval path segments.
+      #
+      # @param segments [Array<String>] Path segments
+      # @return [String, nil] Skill name or nil
+      def self.extract_skill_name(segments)
+        index = segments.rindex('skills')
+        return nil unless index
+
+        remaining = segments[(index + 1)..]
+        return nil if remaining.empty?
 
-        resolve_skills_path(segments) || resolve_workflows_path(segments)
+        remaining[0]
+      end
+
+      # Finds a skill directory within a source path by name.
+      #
+      # @param source_path [String] Root directory containing skill categories
+      # @param skill_name [String] Name of the skill to find
+      # @return [String, nil] Path to the skill directory or nil
+      def self.find_skill_in_source(source_path, skill_name)
+        return nil unless source_path && Dir.exist?(source_path)
+
+        Dir.glob(File.join(source_path, '*')).each do |entry|
+          next unless Dir.exist?(entry)
+
+          candidate = File.join(entry, skill_name)
+          return candidate if Dir.exist?(candidate) && File.exist?(File.join(candidate, 'SKILL.md'))
+        end
+
+        nil
       end
 
       private_class_method def self.resolve_skills_path(segments)
@@ -55,6 +104,13 @@ module SkillBench
         workflow_name = segments[index + 1]
         "workflows/#{workflow_name}" if workflow_name
       end
+
+      private_class_method def self.skill_exists_at?(path)
+        return false unless path
+
+        full_path = path.end_with?('SKILL.md') ? path : File.join(path, 'SKILL.md')
+        File.exist?(full_path)
+      end
     end
   end
 end

data/lib/skill_bench/registry/pack_resolver.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module SkillBench
+  module Registry
+    # Resolves skill paths from the ecosystem registry manifest.
+    # Reads a registry.json (from agent-mcp-runtime) and resolves
+    # pack → tile.json → skill path.
+    class PackResolver
+      # @param registry_path [String] Path to registry.json manifest
+      def initialize(registry_path)
+        @manifest = JSON.parse(File.read(registry_path))
+      end
+
+      # Resolves a skill path within a named pack.
+      #
+      # @param pack_name [String] Pack name (e.g. "rails", "core", "hanami")
+      # @param skill_name [String] Skill name (e.g. "code-review")
+      # @return [String, nil] Absolute path to the skill directory, or nil
+      # @param [Array<Object>] visited
+      def resolve_skill(pack_name, skill_name, visited = [])
+        return nil if visited.include?(pack_name)
+
+        visited += [pack_name]
+
+        pack = @manifest.dig('packs', pack_name)
+        return nil unless pack
+
+        source_path = resolve_source(pack['source'])
+        return nil unless source_path
+
+        tile_path = File.join(source_path, pack['tile'])
+        return nil unless File.exist?(tile_path)
+
+        tile = JSON.parse(File.read(tile_path))
+
+        # 1. Try to resolve directly
+        resolved = resolve_direct(tile, source_path, skill_name)
+        return resolved if resolved
+
+        # 2. Try to resolve via deprecated_skills redirect
+        resolved = resolve_redirect(tile, skill_name, visited)
+        return resolved if resolved
+
+        # 3. Try to resolve via depends_on packs in registry
+        resolve_dependencies(pack, skill_name, visited)
+      end
+
+      # Lists available pack names from the manifest.
+      #
+      # @return [Array<String>] Available pack names
+      def pack_names
+        @manifest.fetch('packs', {}).keys
+      end
+
+      private
+
+      def resolve_direct(tile, source_path, skill_name)
+        skill_entry = tile.dig('skills', skill_name)
+        return nil unless skill_entry
+
+        skill_path = File.join(source_path, skill_entry['path'])
+        resolved = File.expand_path(skill_path)
+        resolved = File.dirname(resolved) if resolved.end_with?('SKILL.md')
+        base = File.expand_path(source_path)
+
+        # Ensure resolved path is inside source directory
+        resolved == base || resolved.start_with?(base + File::SEPARATOR) ? resolved : nil
+      end
+
+      def resolve_redirect(tile, skill_name, visited)
+        dep_entry = tile.dig('deprecated_skills', skill_name)
+        return nil unless dep_entry
+
+        moved_to = dep_entry['moved_to']
+        return nil unless moved_to
+
+        target_pack = find_pack_by_source(moved_to)
+        return nil unless target_pack
+
+        resolve_skill(target_pack, skill_name, visited)
+      end
+
+      def resolve_dependencies(pack, skill_name, visited)
+        depends_on = pack['depends_on']
+        return nil unless depends_on.is_a?(Array)
+
+        depends_on.each do |dep_pack|
+          resolved = resolve_skill(dep_pack, skill_name, visited)
+          return resolved if resolved
+        end
+        nil
+      end
+
+      def find_pack_by_source(source)
+        @manifest.fetch('packs', {}).each do |pack_name, pack_config|
+          if pack_config['source'] == source ||
+             pack_config['source'].to_s.split('/').last == source.to_s.split('/').last
+            return pack_name
+          end
+        end
+        nil
+      end
+
+      def resolve_source(source)
+        return nil unless source.is_a?(String) && !source.empty?
+
+        repo_name = source.split('/').last
+        candidates = [
+          File.expand_path("../#{repo_name}", Dir.pwd),
+          File.expand_path("../../#{repo_name}", Dir.pwd),
+          File.join(Dir.home, '.agent-mcp-runtime', 'cache', repo_name)
+        ]
+        candidates.find { |c| Dir.exist?(c) }
+      end
+    end
+  end
+end