ruby-skill-bench 0.1.0 → 1.0.1

data/lib/skill_bench/services/runner_service.rb

@@ -1,42 +1,49 @@
 # frozen_string_literal: true
 
-require 'json'
-require 'pathname'
-require_relative '../models/eval'
-require_relative '../models/skill'
-require_relative '../models/config'
-require_relative '../models/provider'
-require_relative '../clients/all'
-require_relative 'skill_resolver'
-require_relative '../trend_tracker'
-require_relative '../execution/sandbox'
-require_relative '../execution/context_hydrator'
-require_relative '../execution/source_path_resolver'
-require_relative '../agent/react_agent'
+require_relative '../evaluation/runner'
+require_relative 'eval_resolver'
+require_relative 'skill_resolver_service'
+require_relative 'provider_resolver'
+require_relative 'prompt_builder_service'
+require_relative 'agent_spawner_service'
+require_relative 'context_loader_service'
+require_relative 'judge_params_builder'
+require_relative 'error_response_builder'
+require_relative 'trend_recorder_service'
+require_relative 'output_formatter'
 
 module SkillBench
   module Services
     # Orchestrates the execution of an eval with baseline and context runs.
-    # rubocop:disable Metrics/ClassLength
+    # Coordinates multiple services to resolve entities, spawn agents, and evaluate results.
     class RunnerService
-      # Stand-in provider when no LLM config is available.
-      MOCK_PROVIDER = Struct.new(:name, :runtime, :llm, :merged_config)
-      private_constant :MOCK_PROVIDER
-
+      # Context for evaluation and trend recording
+      EvaluationContext = Struct.new(:evaluation, :skill_context, :baseline_output, :context_output, :provider, :config, keyword_init: true)
       # Runs an eval with the given parameters.
       #
       # @param eval_name [String] Name or path of the eval to run
       # @param skill_names [Array<String>] Names of the skills to use
+      # @param pack [String, nil] Optional pack name for registry-based skill resolution
+      # @param registry_manifest [String, nil] Optional path to registry.json manifest
       # @return [Hash] Result from EvaluationRunner
-      def self.call(eval_name:, skill_names:)
-        new(eval_name: eval_name, skill_names: skill_names).call
+      def self.call(eval_name:, skill_names:, pack: nil, registry_manifest: nil)
+        new(
+          eval_name: eval_name,
+          skill_names: skill_names,
+          pack: pack,
+          registry_manifest: registry_manifest
+        ).call
       end
 
       # @param eval_name [String] Name or path of the eval
       # @param skill_names [Array<String>] Names of the skills
-      def initialize(eval_name:, skill_names:)
+      # @param pack [String, nil] Optional pack name
+      # @param registry_manifest [String, nil] Optional registry.json path
+      def initialize(eval_name:, skill_names:, pack: nil, registry_manifest: nil)
         @eval_name = eval_name
         @skill_names = skill_names
+        @pack = pack
+        @registry_manifest = registry_manifest
       end
 
       # Executes the eval: resolves entities, runs baseline and context, evaluates.
@@ -45,337 +52,99 @@ module SkillBench
       # @raise [Errno::ENOENT] when the eval directory does not exist.
       # @raise [ArgumentError] when a skill cannot be resolved.
       def call
-        evaluation = resolve_eval
-        skills = resolve_skills
-        provider = resolve_provider
+        evaluation = EvalResolver.call(eval_name)
+        skills = SkillResolverService.call(skill_names, pack: pack, registry_manifest: registry_manifest)
+        provider_result = ProviderResolver.call
 
-        config_result = resolve_provider_config(provider)
-        return config_error_result(config_result[:error], evaluation, provider) unless config_result[:success]
+        return config_error_result(provider_result[:error], evaluation, provider_result[:provider]) unless provider_result[:success]
 
-        config = config_result[:config]
-        baseline_prompt = build_baseline_system_prompt
+        provider = provider_result[:provider]
+        config = provider_result[:config]
 
-        baseline_output = spawn_agent(evaluation, baseline_prompt, provider, config)
+        baseline_output = run_baseline_agent(evaluation, provider, config)
         return agent_error_result(baseline_output, 'baseline', evaluation, provider) if baseline_output[:status] == :error
 
-        skill_context = load_combined_skill_context(skills)
+        skill_context = ContextLoaderService.call(skills)
         return empty_context_error_result(evaluation, provider) if skill_context.strip.empty?
 
-        context_prompt = build_context_system_prompt(evaluation, skills)
-        context_output = spawn_agent(evaluation, context_prompt, provider, config)
+        context_output = run_context_agent(evaluation, skills, skill_context, provider, config)
         return agent_error_result(context_output, 'context', evaluation, provider) if context_output[:status] == :error
 
-        criteria = evaluation.criteria
-
-        judge_params = build_judge_params(provider, config)
-
-        result = Evaluation::Runner.call(
-          task: evaluation.task,
-          criteria: criteria,
+        context = EvaluationContext.new(
+          evaluation: evaluation,
           skill_context: skill_context,
-          baseline_output: format_output(baseline_output),
-          context_output: format_output(context_output),
-          judge_params: judge_params
+          baseline_output: baseline_output,
+          context_output: context_output,
+          provider: provider,
+          config: config
         )
-
-        return enrich_error_result(result, evaluation, provider) unless result[:success]
-
-        trend_result = record_and_compute_trend(result)
-        return enrich_error_result(trend_result, evaluation, provider) unless trend_result[:success]
-
-        {
-          success: true,
-          eval_name: eval_name,
-          skill_name: skill_names.join(', '),
-          provider_name: provider.name,
-          response: result[:response].merge(
-            trend: trend_result[:trend],
-            baseline_iterations: baseline_output[:iterations] || [],
-            context_iterations: context_output[:iterations] || []
-          )
-        }
+        evaluate_and_record_trend(context)
       end
 
       private
 
-      attr_reader :eval_name, :skill_names
-
-      def resolve_eval
-        eval_path = eval_name.include?('/') ? eval_name : "evals/#{eval_name}"
-        SkillBench::Models::Eval.load(eval_path)
-      end
-
-      def resolve_skills
-        skill_names.map { |name| Services::SkillResolver.call(name) }
-      end
-
-      def resolve_provider_config(provider)
-        { success: true, config: provider.merged_config }
-      rescue ArgumentError => e
-        { success: false, error: e }
-      end
-
-      # Safely calls merged_config, returning nil on any error.
-      #
-      # @param provider [Object] The provider to query.
-      # @return [Hash, nil] The merged config or nil.
-      def safe_merged_config(provider)
-        provider.merged_config
-      rescue StandardError
-        nil
-      end
-
-      def resolve_provider
-        config = SkillBench::Models::Config.load
-        provider = config.to_provider
-        return provider if provider
-
-        warn 'Config load failed, using mock provider'
-        MOCK_PROVIDER.new('mock', 'mock', 'mock', {})
-      end
-
-      # Spawns the LLM agent with the given system prompt.
-      #
-      # @param evaluation [SkillBench::Models::Eval] The eval being run.
-      # @param system_prompt [String] The system prompt for the agent.
-      # @param provider [Object] The resolved provider.
-      # @param config [Hash, nil] Provider config.
-      # @return [Hash] Agent response with result, status, runtime, usage, raw_response, iterations.
-      def spawn_agent(evaluation, system_prompt, provider, config)
-        return { result: 'mock result', status: :success, iterations: [] } if provider.name == 'mock'
-
-        client_params = build_client_params(provider, config)
-
-        max_iterations = config&.[](:max_iterations) || config&.[]('max_iterations') || 25
-
-        Execution::Sandbox.run(evaluation.path) do |sandbox|
-          agent_result = Agent::ReactAgent.call(
-            system_prompt: system_prompt,
-            initial_prompt: evaluation.task,
-            working_dir: sandbox.path,
-            container_id: sandbox.container_id,
-            client_params: client_params,
-            max_iterations: max_iterations
-          )
-
-          status = agent_result[:success] ? :success : :error
-          final_answer = agent_result.dig(:response, :content) || ''
-          diff = Execution::Sandbox.capture_diff(sandbox.path)
-          iterations = agent_result.dig(:response, :iterations) || []
-
-          output = [final_answer, diff].reject(&:empty?).join("\n\n")
-
-          {
-            result: output,
-            status: status,
-            runtime: provider.runtime,
-            usage: {},
-            raw_response: agent_result,
-            iterations: iterations
-          }
-        end
-      end
-
-      # Builds client parameters for the ReactAgent.
-      #
-      # @param provider [Object] The resolved provider.
-      # @param config [Hash, nil] Provider config.
-      # @return [Hash] Client parameters.
-      def build_client_params(provider, config)
-        config ||= safe_merged_config(provider)
-        return {} unless config
-
-        params = config.dup
-        params[:model] ||= provider.llm
-        params[:provider] = provider.runtime.to_sym
-        params
-      rescue StandardError
-        {}
-      end
-
-      # Builds the baseline system prompt (no skill context).
-      #
-      # @return [String] The baseline system prompt.
-      def build_baseline_system_prompt
-        <<~PROMPT
-          You are an expert Ruby on Rails developer. Your job is to read the task,
-          modify the codebase using the tools provided to meet the requirements,
-          and then explain what you did.
-        PROMPT
-      end
+      attr_reader :eval_name, :skill_names, :pack, :registry_manifest
 
-      # Builds the context-aware system prompt based on eval metadata.
-      #
-      # For `skill_bundle_xml` context mode, combines SKILL.md with source code
-      # via ContextHydrator. Falls back to SKILL.md-only if source is unavailable.
-      #
-      # @param evaluation [SkillBench::Models::Eval] The eval being run.
-      # @param skills [Array<SkillBench::Models::Skill>] Resolved skills.
-      # @return [String] The context system prompt.
-      def build_context_system_prompt(evaluation, skills)
-        skill_md_content = load_combined_skill_context(skills)
-        return skill_md_content unless evaluation.metadata['context_mode'] == 'skill_bundle_xml'
-
-        source_path = resolve_source_path(evaluation)
-        return skill_md_content unless source_path
-
-        xml_result = Execution::ContextHydrator.call(source_path: source_path, base_path: Pathname.new(Dir.pwd))
-        hydrator_response = xml_result[:response]
-        xml_context = hydrator_response[:context]
-        return skill_md_content unless xml_result[:success] && !xml_context.empty?
-
-        <<~PROMPT
-          You are an expert Ruby on Rails developer.
-          You have access to a skill file and source code wrapped in <agent_context> tags.
-          Use the skill instructions and the provided source code to solve the task.
-
-          ## Skill Instructions
-          #{skill_md_content}
-
-          ## Source Code
-          #{xml_context}
-        PROMPT
+      def config_error_result(error, evaluation, provider)
+        ErrorResponseBuilder.config_error(error, evaluation, provider, skill_names)
       end
 
-      # Resolves the source path for context hydration.
-      #
-      # Tries the eval's `source/` subdirectory first, then falls back to
-      # SourcePathResolver inference.
-      #
-      # @param evaluation [SkillBench::Models::Eval] The eval being run.
-      # @return [String, nil] The resolved source path, or nil if not found.
-      def resolve_source_path(evaluation)
-        eval_path = evaluation.path
-        eval_source = File.join(eval_path, 'source')
-        return eval_source if Dir.exist?(eval_source)
-
-        inferred = Execution::SourcePathResolver.call(eval_folder_path: eval_path.to_s)
-        inferred if inferred && Dir.exist?(inferred)
+      def agent_error_result(result, phase, evaluation, provider)
+        ErrorResponseBuilder.agent_error(result, phase, evaluation, provider, skill_names)
       end
 
-      # Returns an error result when skill context is empty.
-      #
-      # @param evaluation [SkillBench::Models::Eval] The eval being run.
-      # @param provider [Object] The resolved provider.
-      # @return [Hash] Error result with metadata.
       def empty_context_error_result(evaluation, provider)
-        {
-          success: false,
-          response: {
-            error: {
-              message: 'Skill context is empty. Ensure SKILL.md exists and has content.'
-            }
-          },
-          eval_name: evaluation.name,
-          skill_name: skill_names.join(', '),
-          provider_name: provider.name
-        }
+        ErrorResponseBuilder.empty_context_error(evaluation, provider, skill_names)
       end
 
-      def load_combined_skill_context(skills)
-        return '' if skills.nil? || skills.empty?
+      def enrich_error_result(result, evaluation, provider)
+        ErrorResponseBuilder.enrich_error(result, evaluation, provider, skill_names)
+      end
 
-        contexts = skills.map { |skill| load_skill_context(skill) }
-        contexts.reject(&:empty?).join("\n\n#{'=' * 40}\n\n")
+      def run_baseline_agent(evaluation, provider, config)
+        baseline_prompt = PromptBuilderService.build_baseline
+        AgentSpawnerService.call(evaluation, baseline_prompt, provider, config)
       end
 
-      def load_skill_context(skill)
-        skill_md = File.join(skill.path, 'SKILL.md')
-        File.exist?(skill_md) ? File.read(skill_md) : ''
+      def run_context_agent(evaluation, skills, skill_context, provider, config)
+        context_prompt = PromptBuilderService.build_context(evaluation, skills, skill_context)
+        AgentSpawnerService.call(evaluation, context_prompt, provider, config)
       end
 
-      def build_judge_params(provider, config)
-        return {} if provider.name == 'mock'
+      def evaluate_and_record_trend(context)
+        evaluation = context.evaluation
+        provider = context.provider
+        config = context.config
 
-        config ||= safe_merged_config(provider)
-        return {} unless config
+        criteria = evaluation.criteria
+        judge_params = JudgeParamsBuilder.call(provider, config)
 
-        {
-          api_key: config[:api_key],
-          model: config[:model] || provider.llm,
-          provider: provider.runtime.to_sym
-        }
-      rescue StandardError
-        {}
-      end
+        result = Evaluation::Runner.call(
+          task: evaluation.task,
+          criteria: criteria,
+          skill_context: context.skill_context,
+          baseline_output: OutputFormatter.call(context.baseline_output),
+          context_output: OutputFormatter.call(context.context_output),
+          judge_params: judge_params
+        )
 
-      def format_output(agent_result)
-        agent_result[:result].to_s
-      end
+        return enrich_error_result(result, evaluation, provider) unless result[:success]
 
-      def agent_error_result(result, phase, evaluation, provider)
-        raw = result[:raw_response]
-        error_msg = raw&.dig(:response, :error, :message) || raw&.dig(:error, :message) || 'unknown error'
-        {
-          success: false,
-          response: {
-            error: {
-              message: "#{phase.capitalize} agent failed: #{error_msg}"
-            }
-          },
-          eval_name: evaluation.name,
-          skill_name: skill_names.join(', '),
-          provider_name: provider.name
-        }
-      end
+        trend_result = TrendRecorderService.call(result, eval_name, skill_names)
+        return enrich_error_result(trend_result, evaluation, provider) unless trend_result[:success]
 
-      def config_error_result(error, evaluation, provider)
         {
-          success: false,
-          response: {
-            error: {
-              message: "Configuration error: #{error.message}"
-            }
-          },
-          eval_name: evaluation.name,
-          skill_name: skill_names.join(', '),
-          provider_name: provider.name
-        }
-      end
-
-      def enrich_error_result(result, evaluation, provider)
-        result.merge(
-          eval_name: evaluation.name,
+          success: true,
+          eval_name: eval_name,
           skill_name: skill_names.join(', '),
-          provider_name: provider.name
-        )
-      end
-
-      def record_and_compute_trend(result)
-        tracker = TrendTracker.new
-        enriched = result.merge(eval_name: eval_name, skill_names: skill_names)
-        trend = tracker.trend_for(enriched)
-        record_result = tracker.record(enriched)
-
-        record_success = record_result.is_a?(Hash) && record_result[:success]
-        unless record_success
-          message = if record_result.is_a?(Hash)
-                      record_result.dig(:response, :error, :message) ||
-                        record_result.dig(:error, :message) ||
-                        'Unknown error'
-                    else
-                      'Unexpected record response'
-                    end
-          SkillBench::ErrorLogger.log_error(
-            StandardError.new(message),
-            "Trend tracking record failed for eval #{eval_name}"
+          provider_name: provider.name,
+          response: result[:response].merge(
+            trend: trend_result[:trend],
+            baseline_iterations: context.baseline_output[:iterations] || [],
+            context_iterations: context.context_output[:iterations] || []
           )
-          return {
-            success: false,
-            response: {
-              error: {
-                message: "Trend tracking record failed: #{message}",
-                record_result: record_result
-              }
-            }
-          }
-        end
-        { success: true, trend: trend }
-      rescue StandardError => e
-        SkillBench::ErrorLogger.log_error(e, 'Trend tracking failed')
-        { success: false, response: { error: { message: e.message } } }
+        }
       end
-      # rubocop:enable Metrics/ClassLength
     end
   end
 end

data/lib/skill_bench/services/skill_resolver.rb

@@ -48,7 +48,21 @@ module SkillBench
         cwd = File.expand_path(Dir.pwd)
         cwd_with_sep = cwd + File::SEPARATOR
 
-        raise(ArgumentError, "Skill path escapes project boundary: #{identifier}") unless absolute_path == cwd || absolute_path.start_with?(cwd_with_sep)
+        allowed = absolute_path == cwd || absolute_path.start_with?(cwd_with_sep)
+        unless allowed
+          sources = SkillBench::Config.skill_sources
+          if sources.is_a?(Hash)
+            sources.each_value do |source_path|
+              abs_src = File.expand_path(source_path)
+              if absolute_path == abs_src || absolute_path.start_with?(abs_src + File::SEPARATOR)
+                allowed = true
+                break
+              end
+            end
+          end
+        end
+
+        raise(ArgumentError, "Skill path escapes project boundary: #{identifier}") unless allowed
 
         skill_md = File.join(normalized_path, 'SKILL.md')
 
@@ -57,21 +71,35 @@ module SkillBench
         raise(ArgumentError, "Skill not found: #{identifier}")
       end
 
-      # Resolves a skill by name using recursive discovery.
-      #
-      # @return [SkillBench::Models::Skill] The resolved skill
-      # @raise [ArgumentError] if no skill with matching name found
       def resolve_by_name
-        skills = Models::Skill.discover(base_path)
+        skills = discover_all_skills
         matches = skills.select { |skill| skill.name == identifier }
 
+        validate_matches!(matches)
+
+        matches.first
+      end
+
+      def discover_all_skills
+        skills = Models::Skill.discover(base_path)
+
+        sources = SkillBench::Config.skill_sources
+        if sources.is_a?(Hash)
+          sources.each_value do |source_path|
+            skills += Models::Skill.discover(source_path) if Dir.exist?(source_path)
+          end
+        end
+
+        skills
+      end
+
+      def validate_matches!(matches)
         if matches.empty?
           raise(ArgumentError, "Skill not found: #{identifier}")
         elsif matches.size > 1
-          raise(ArgumentError, "Multiple skills found with name '#{identifier}': #{matches.map(&:path).join(', ')}")
+          matches.uniq! { |m| File.expand_path(m.path) }
+          raise(ArgumentError, "Multiple skills found with name '#{identifier}': #{matches.map(&:path).join(', ')}") if matches.size > 1
         end
-
-        matches.first
       end
     end
   end

data/lib/skill_bench/services/skill_resolver_service.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative '../models/skill'
+require_relative 'skill_resolver'
+require_relative '../registry/pack_resolver'
+
+module SkillBench
+  module Services
+    # Resolves skills from names, supporting both direct resolution and pack-based resolution.
+    class SkillResolverService
+      # Default registry manifest path relative to the current working directory.
+      DEFAULT_REGISTRY_MANIFEST = '../agent-mcp-runtime/registry.json'
+      private_constant :DEFAULT_REGISTRY_MANIFEST
+
+      # Resolves skills from names.
+      #
+      # @param skill_names [Array<String>] Names of the skills to resolve
+      # @param pack [String, nil] Optional pack name for registry-based skill resolution
+      # @param registry_manifest [String, nil] Optional path to registry.json manifest
+      # @return [Array<SkillBench::Models::Skill>] The resolved skills
+      # @raise [ArgumentError] when a skill cannot be resolved
+      def self.call(skill_names, pack: nil, registry_manifest: nil)
+        new(skill_names, pack: pack, registry_manifest: registry_manifest).call
+      end
+
+      # @param skill_names [Array<String>] Names of the skills
+      # @param pack [String, nil] Optional pack name
+      # @param registry_manifest [String, nil] Optional registry.json path
+      def initialize(skill_names, pack: nil, registry_manifest: nil)
+        @skill_names = skill_names
+        @pack = pack
+        @registry_manifest = registry_manifest
+      end
+
+      # Resolves the skills from names.
+      #
+      # @return [Array<SkillBench::Models::Skill>] The resolved skills
+      # @raise [ArgumentError] when a skill cannot be resolved
+      def call
+        return @call if defined?(@call)
+
+        @call = if @pack && !@pack.empty?
+                  resolve_pack_skills
+                else
+                  @skill_names.map { |name| Services::SkillResolver.call(name) }
+                end
+      end
+
+      private
+
+      attr_reader :skill_names, :pack, :registry_manifest
+
+      def resolve_pack_skills
+        manifest_path = registry_manifest || DEFAULT_REGISTRY_MANIFEST
+        manifest_absolute = File.expand_path(manifest_path, Dir.pwd)
+
+        raise ArgumentError, "Registry manifest not found: #{manifest_path}" unless File.exist?(manifest_absolute)
+
+        resolver = Registry::PackResolver.new(manifest_absolute)
+
+        skill_names.map do |skill_name|
+          path = resolver.resolve_skill(pack, skill_name)
+          raise ArgumentError, "Skill '#{skill_name}' not found in pack '#{pack}'" unless path
+
+          Models::Skill.new(name: skill_name, path: path)
+        end
+      end
+    end
+  end
+end

data/lib/skill_bench/services/source_path_resolver_service.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative '../execution/source_path_resolver'
+
+module SkillBench
+  module Services
+    # Resolves the source path for context hydration.
+    class SourcePathResolverService
+      # Resolves the source path for context hydration.
+      #
+      # Tries the eval's `source/` subdirectory first, then falls back to
+      # SourcePathResolver inference.
+      #
+      # @param evaluation [SkillBench::Models::Eval] The eval being run
+      # @return [String, nil] The resolved source path, or nil if not found
+      def self.call(evaluation)
+        new(evaluation).call
+      end
+
+      # @param evaluation [SkillBench::Models::Eval] The eval being run
+      def initialize(evaluation)
+        @evaluation = evaluation
+      end
+
+      # Resolves the source path for context hydration.
+      #
+      # Tries the eval's `source/` subdirectory first, then falls back to
+      # SourcePathResolver inference.
+      #
+      # @return [String, nil] The resolved source path, or nil if not found
+      def call
+        eval_path = @evaluation.path
+        eval_source = File.join(eval_path, 'source')
+        return eval_source if Dir.exist?(eval_source)
+
+        sources = SkillBench::Config.skill_sources || {}
+        inferred = Execution::SourcePathResolver.call(
+          eval_folder_path: eval_path.to_s,
+          skill_sources: sources
+        )
+        inferred if inferred && Dir.exist?(inferred)
+      end
+    end
+  end
+end