ruby-skill-bench 0.1.0 → 1.1.0

data/lib/skill_bench/services/skill_resolver.rb

@@ -48,7 +48,21 @@ module SkillBench
         cwd = File.expand_path(Dir.pwd)
         cwd_with_sep = cwd + File::SEPARATOR
 
-        raise(ArgumentError, "Skill path escapes project boundary: #{identifier}") unless absolute_path == cwd || absolute_path.start_with?(cwd_with_sep)
+        allowed = absolute_path == cwd || absolute_path.start_with?(cwd_with_sep)
+        unless allowed
+          sources = SkillBench::Config.skill_sources
+          if sources.is_a?(Hash)
+            sources.each_value do |source_path|
+              abs_src = File.expand_path(source_path)
+              if absolute_path == abs_src || absolute_path.start_with?(abs_src + File::SEPARATOR)
+                allowed = true
+                break
+              end
+            end
+          end
+        end
+
+        raise(ArgumentError, "Skill path escapes project boundary: #{identifier}") unless allowed
 
         skill_md = File.join(normalized_path, 'SKILL.md')
 
@@ -57,21 +71,35 @@ module SkillBench
         raise(ArgumentError, "Skill not found: #{identifier}")
       end
 
-      # Resolves a skill by name using recursive discovery.
-      #
-      # @return [SkillBench::Models::Skill] The resolved skill
-      # @raise [ArgumentError] if no skill with matching name found
       def resolve_by_name
-        skills = Models::Skill.discover(base_path)
+        skills = discover_all_skills
         matches = skills.select { |skill| skill.name == identifier }
 
+        validate_matches!(matches)
+
+        matches.first
+      end
+
+      def discover_all_skills
+        skills = Models::Skill.discover(base_path)
+
+        sources = SkillBench::Config.skill_sources
+        if sources.is_a?(Hash)
+          sources.each_value do |source_path|
+            skills += Models::Skill.discover(source_path) if Dir.exist?(source_path)
+          end
+        end
+
+        skills
+      end
+
+      def validate_matches!(matches)
         if matches.empty?
           raise(ArgumentError, "Skill not found: #{identifier}")
         elsif matches.size > 1
-          raise(ArgumentError, "Multiple skills found with name '#{identifier}': #{matches.map(&:path).join(', ')}")
+          matches.uniq! { |m| File.expand_path(m.path) }
+          raise(ArgumentError, "Multiple skills found with name '#{identifier}': #{matches.map(&:path).join(', ')}") if matches.size > 1
         end
-
-        matches.first
       end
     end
   end

data/lib/skill_bench/services/skill_resolver_service.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative '../models/skill'
+require_relative 'skill_resolver'
+require_relative '../registry/pack_resolver'
+
+module SkillBench
+  module Services
+    # Resolves skills from names, supporting both direct resolution and pack-based resolution.
+    class SkillResolverService
+      # Default registry manifest path relative to the current working directory.
+      DEFAULT_REGISTRY_MANIFEST = '../agent-mcp-runtime/registry.json'
+      private_constant :DEFAULT_REGISTRY_MANIFEST
+
+      # Resolves skills from names.
+      #
+      # @param skill_names [Array<String>] Names of the skills to resolve
+      # @param pack [String, nil] Optional pack name for registry-based skill resolution
+      # @param registry_manifest [String, nil] Optional path to registry.json manifest
+      # @return [Array<SkillBench::Models::Skill>] The resolved skills
+      # @raise [ArgumentError] when a skill cannot be resolved
+      def self.call(skill_names, pack: nil, registry_manifest: nil)
+        new(skill_names, pack: pack, registry_manifest: registry_manifest).call
+      end
+
+      # @param skill_names [Array<String>] Names of the skills
+      # @param pack [String, nil] Optional pack name
+      # @param registry_manifest [String, nil] Optional registry.json path
+      def initialize(skill_names, pack: nil, registry_manifest: nil)
+        @skill_names = skill_names
+        @pack = pack
+        @registry_manifest = registry_manifest
+      end
+
+      # Resolves the skills from names.
+      #
+      # @return [Array<SkillBench::Models::Skill>] The resolved skills
+      # @raise [ArgumentError] when a skill cannot be resolved
+      def call
+        return @call if defined?(@call)
+
+        @call = if @pack && !@pack.empty?
+                  resolve_pack_skills
+                else
+                  @skill_names.map { |name| Services::SkillResolver.call(name) }
+                end
+      end
+
+      private
+
+      attr_reader :skill_names, :pack, :registry_manifest
+
+      def resolve_pack_skills
+        manifest_path = registry_manifest || DEFAULT_REGISTRY_MANIFEST
+        manifest_absolute = File.expand_path(manifest_path, Dir.pwd)
+
+        raise ArgumentError, "Registry manifest not found: #{manifest_path}" unless File.exist?(manifest_absolute)
+
+        resolver = Registry::PackResolver.new(manifest_absolute)
+
+        skill_names.map do |skill_name|
+          path = resolver.resolve_skill(pack, skill_name)
+          raise ArgumentError, "Skill '#{skill_name}' not found in pack '#{pack}'" unless path
+
+          Models::Skill.new(name: skill_name, path: path)
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/services/source_path_resolver_service.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative '../execution/source_path_resolver'
+
+module SkillBench
+  module Services
+    # Resolves the source path for context hydration.
+    class SourcePathResolverService
+      # Resolves the source path for context hydration.
+      #
+      # Tries the eval's `source/` subdirectory first, then falls back to
+      # SourcePathResolver inference.
+      #
+      # @param evaluation [SkillBench::Models::Eval] The eval being run
+      # @return [String, nil] The resolved source path, or nil if not found
+      def self.call(evaluation)
+        new(evaluation).call
+      end
+
+      # @param evaluation [SkillBench::Models::Eval] The eval being run
+      def initialize(evaluation)
+        @evaluation = evaluation
+      end
+
+      # Resolves the source path for context hydration.
+      #
+      # Tries the eval's `source/` subdirectory first, then falls back to
+      # SourcePathResolver inference.
+      #
+      # @return [String, nil] The resolved source path, or nil if not found
+      def call
+        eval_path = @evaluation.path
+        eval_source = File.join(eval_path, 'source')
+        return eval_source if Dir.exist?(eval_source)
+
+        sources = SkillBench::Config.skill_sources || {}
+        inferred = Execution::SourcePathResolver.call(
+          eval_folder_path: eval_path.to_s,
+          skill_sources: sources
+        )
+        inferred if inferred && Dir.exist?(inferred)
+      end
+    end
+  end
+end

data/lib/skill_bench/services/trend_recorder_service.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative '../trend_tracker'
+
+module SkillBench
+  module Services
+    # Records evaluation results and computes trends.
+    class TrendRecorderService
+      # Records evaluation results and computes trends.
+      #
+      # @param result [Hash] The evaluation result from Evaluation::Runner
+      # @param eval_name [String] Name of the eval
+      # @param skill_names [Array<String>] Names of the skills
+      # @return [Hash] Result with success status and trend data
+      def self.call(result, eval_name, skill_names)
+        new(result, eval_name, skill_names).call
+      end
+
+      # @param result [Hash] The evaluation result from Evaluation::Runner
+      # @param eval_name [String] Name of the eval
+      # @param skill_names [Array<String>] Names of the skills
+      def initialize(result, eval_name, skill_names)
+        @result = result
+        @eval_name = eval_name
+        @skill_names = skill_names
+      end
+
+      # Records evaluation results and computes trends.
+      #
+      # @return [Hash] Result with success status and trend data
+      def call
+        tracker = TrendTracker.new
+        enriched = @result.merge(eval_name: @eval_name, skill_names: @skill_names)
+        trend = tracker.trend_for(enriched)
+        record_result = tracker.record(enriched)
+
+        record_success = record_result.is_a?(Hash) && record_result[:success]
+        unless record_success
+          message = if record_result.is_a?(Hash)
+                      record_result.dig(:response, :error, :message) ||
+                        record_result.dig(:error, :message) ||
+                        'Unknown error'
+                    else
+                      'Unexpected record response'
+                    end
+          SkillBench::ErrorLogger.log_error(
+            StandardError.new(message),
+            "Trend tracking record failed for eval #{@eval_name}"
+          )
+          return {
+            success: false,
+            response: {
+              error: {
+                message: "Trend tracking record failed: #{message}",
+                record_result: record_result
+              }
+            }
+          }
+        end
+        { success: true, trend: trend }
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'Trend tracking failed')
+        { success: false, response: { error: { message: e.message } } }
+      end
+    end
+  end
+end

data/lib/skill_bench/services/variant_parser.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Services
+    # Parses variant specifications for skill comparison.
+    class VariantParser
+      # Parses a variant specification string.
+      #
+      # @param spec [String] Variant spec (e.g., "pack:rails" or "/path/to/skill")
+      # @return [Hash] Parsed variant with :type (:pack or :path) and corresponding key
+      def self.call(spec)
+        new(spec).call
+      end
+
+      # @param spec [String] Variant spec (e.g., "pack:rails" or "/path/to/skill")
+      def initialize(spec)
+        @spec = spec
+      end
+
+      # Parses the variant specification.
+      #
+      # @return [Hash] Parsed variant with :type (:pack or :path) and corresponding key
+      def call
+        if @spec.start_with?('pack:')
+          { type: :pack, name: @spec.sub('pack:', '') }
+        else
+          { type: :path, path: @spec }
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/services/variant_resolver.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative '../registry/pack_resolver'
+require_relative 'runner_service'
+require_relative 'manifest_finder'
+
+module SkillBench
+  module Services
+    # Resolves skill paths from variant specifications.
+    class VariantResolver
+      # Resolves skill paths from a variant specification.
+      #
+      # @param variant [Hash] Parsed variant from VariantParser
+      # @param skill_name [String] Name of the skill to resolve
+      # @param manifest_path [String, nil] Optional path to registry manifest
+      # @return [Array<String>] Array of skill paths
+      # @raise [ArgumentError] when skill cannot be resolved
+      def self.call(variant, skill_name, manifest_path: nil)
+        new(variant, skill_name, manifest_path: manifest_path).call
+      end
+
+      # @param variant [Hash] Parsed variant from VariantParser
+      # @param skill_name [String] Name of the skill to resolve
+      # @param manifest_path [String, nil] Optional path to registry manifest
+      def initialize(variant, skill_name, manifest_path: nil)
+        @variant = variant
+        @skill_name = skill_name
+        @manifest_path = manifest_path
+      end
+
+      # Resolves skill paths from the variant specification.
+      #
+      # @return [Array<String>] Array of skill paths
+      # @raise [ArgumentError] when skill cannot be resolved or variant type is unknown
+      def call
+        case @variant[:type]
+        when :pack
+          resolve_pack_skill
+        when :path
+          [@variant[:path]]
+        else
+          raise ArgumentError, "Unknown variant type: #{@variant[:type]}, variant: #{@variant.inspect}"
+        end
+      end
+
+      private
+
+      # Resolves a skill from a pack.
+      #
+      # @return [Array<String>] Array containing the resolved skill path
+      # @raise [ArgumentError] when skill is not found in pack
+      def resolve_pack_skill
+        pack_name = @variant[:name]
+        manifest = @manifest_path || ManifestFinder.call
+        resolver = Registry::PackResolver.new(manifest)
+        resolved = resolver.resolve_skill(pack_name, @skill_name)
+        raise ArgumentError, "Skill '#{@skill_name}' not found in pack '#{pack_name}'" unless resolved
+
+        [resolved]
+      end
+    end
+  end
+end

data/lib/skill_bench/tools/run_command.rb

@@ -4,27 +4,12 @@ require 'open3'
 require 'timeout'
 require 'shellwords'
 require_relative '../config'
+require_relative '../constants'
 
 module SkillBench
   module Tools
     # Handles executing a shell command within the working directory.
     class RunCommand
-      # Commands that are always blocked even if listed in allowed_commands,
-      # because they can be used to escape the sandbox or execute arbitrary code.
-      DANGEROUS_COMMANDS = %w[
-        bash sh zsh fish dash ksh csh tcsh
-        python python3 python2 ruby perl node
-        php lua tcl wish
-        curl wget nc ncat socat
-        eval exec
-        sudo su doas
-        chmod chown mount umount
-        dd mkfs fdisk parted
-        insmod rmmod modprobe
-        systemctl service
-        passwd useradd userdel groupadd groupdel
-      ].freeze
-
       # @return [Hash] The tool definition for the LLM API.
       def self.definition
         {
@@ -59,7 +44,7 @@ module SkillBench
         return 'Error: Empty command.' if argv.empty?
 
         base_cmd = argv.first
-        return "Error: Command '#{base_cmd}' is blocked for security reasons." if DANGEROUS_COMMANDS.include?(base_cmd)
+        return "Error: Command '#{base_cmd}' is blocked for security reasons." if Constants::Tools::DANGEROUS_COMMANDS.include?(base_cmd)
 
         allowed = SkillBench::Config.allowed_commands
         return 'Error: No allowed commands configured. Set allowed_commands in skill-bench.json or use --mode mock.' if allowed.nil?

data/lib/skill_bench/version.rb

@@ -2,5 +2,5 @@
 
 module SkillBench
   # The current gem version.
-  VERSION = '0.1.0'
+  VERSION = '1.1.0'
 end

data/lib/skill_bench.rb

@@ -8,6 +8,7 @@
 
 # Core modules
 require_relative 'skill_bench/version'
+require_relative 'skill_bench/constants'
 require_relative 'skill_bench/dimension'
 require_relative 'skill_bench/criteria'
 require_relative 'skill_bench/delta_report'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby-skill-bench
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Ismael Marin
@@ -119,6 +119,7 @@ files:
 - lib/skill_bench/agent/runner.rb
 - lib/skill_bench/agent/summary.rb
 - lib/skill_bench/cli.rb
+- lib/skill_bench/cli/compare_command.rb
 - lib/skill_bench/cli/eval/eval_command_registry.rb
 - lib/skill_bench/cli/eval/eval_commands.rb
 - lib/skill_bench/cli/eval/eval_options.rb
@@ -139,12 +140,14 @@ files:
 - lib/skill_bench/clients/providers/deepseek.rb
 - lib/skill_bench/clients/providers/gemini.rb
 - lib/skill_bench/clients/providers/groq.rb
+- lib/skill_bench/clients/providers/mock.rb
 - lib/skill_bench/clients/providers/null_client.rb
 - lib/skill_bench/clients/providers/ollama.rb
 - lib/skill_bench/clients/providers/openai.rb
 - lib/skill_bench/clients/providers/opencode.rb
 - lib/skill_bench/clients/providers/openrouter.rb
 - lib/skill_bench/clients/request_builder.rb
+- lib/skill_bench/clients/response_builder.rb
 - lib/skill_bench/clients/response_error_handler.rb
 - lib/skill_bench/clients/response_parser.rb
 - lib/skill_bench/clients/retry_handler.rb
@@ -160,6 +163,7 @@ files:
 - lib/skill_bench/config/facade_writers.rb
 - lib/skill_bench/config/json_loader.rb
 - lib/skill_bench/config/store.rb
+- lib/skill_bench/constants.rb
 - lib/skill_bench/criteria.rb
 - lib/skill_bench/delta_report.rb
 - lib/skill_bench/dimension.rb
@@ -191,21 +195,40 @@ files:
 - lib/skill_bench/output_formatter.rb
 - lib/skill_bench/package_verifier.rb
 - lib/skill_bench/rails/skill_templates.rb
+- lib/skill_bench/registry/pack_resolver.rb
 - lib/skill_bench/runner.rb
+- lib/skill_bench/services/agent_spawner_service.rb
+- lib/skill_bench/services/compare_option_parser.rb
+- lib/skill_bench/services/comparison_reporter.rb
+- lib/skill_bench/services/comparison_runner.rb
+- lib/skill_bench/services/context_loader_service.rb
 - lib/skill_bench/services/delta_table_formatter.rb
+- lib/skill_bench/services/error_response_builder.rb
+- lib/skill_bench/services/eval_resolver.rb
+- lib/skill_bench/services/exit_code_calculator.rb
 - lib/skill_bench/services/feedback_generator.rb
 - lib/skill_bench/services/formatting_helpers.rb
 - lib/skill_bench/services/iteration_formatter.rb
 - lib/skill_bench/services/json_formatter.rb
+- lib/skill_bench/services/judge_params_builder.rb
 - lib/skill_bench/services/judge_score_parser_service.rb
 - lib/skill_bench/services/junit_formatter.rb
+- lib/skill_bench/services/manifest_finder.rb
 - lib/skill_bench/services/option_parser_service.rb
+- lib/skill_bench/services/output_formatter.rb
 - lib/skill_bench/services/output_persistence_service.rb
+- lib/skill_bench/services/prompt_builder_service.rb
+- lib/skill_bench/services/provider_resolver.rb
 - lib/skill_bench/services/result_printer_service.rb
 - lib/skill_bench/services/runner_service.rb
 - lib/skill_bench/services/skill_resolver.rb
+- lib/skill_bench/services/skill_resolver_service.rb
+- lib/skill_bench/services/source_path_resolver_service.rb
 - lib/skill_bench/services/template_registry.rb
 - lib/skill_bench/services/template_registry/category_data.rb
+- lib/skill_bench/services/trend_recorder_service.rb
+- lib/skill_bench/services/variant_parser.rb
+- lib/skill_bench/services/variant_resolver.rb
 - lib/skill_bench/task.rb
 - lib/skill_bench/task/evaluator.rb
 - lib/skill_bench/task/file_reader.rb
@@ -241,7 +264,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.11
+rubygems_version: 4.0.12
 specification_version: 4
 summary: The evaluation engine for AI Agent Skills benchmarking.
 test_files: []