ruby-skill-bench 0.1.0 → 1.0.1

data/lib/skill_bench/services/judge_params_builder.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Services
+    # Builds judge parameters from provider configuration.
+    class JudgeParamsBuilder
+      # Builds judge parameters from provider configuration.
+      #
+      # @param provider [Object] The resolved provider
+      # @param config [Hash, nil] Provider config
+      # @return [Hash] Judge parameters with api_key, model, and provider
+      def self.call(provider, config)
+        new(provider, config).call
+      end
+
+      # @param provider [Object] The resolved provider
+      # @param config [Hash, nil] Provider config
+      def initialize(provider, config)
+        @provider = provider
+        @config = config
+      end
+
+      # Builds judge parameters from provider configuration.
+      #
+      # @return [Hash] Judge parameters with api_key, model, and provider
+      def call
+        return { provider: :mock } if @provider.name == 'mock'
+
+        config = @config || safe_merged_config
+        return {} unless config
+
+        {
+          api_key: config[:api_key],
+          model: config[:model] || @provider.llm,
+          provider: @provider.runtime.to_sym
+        }
+      rescue KeyError, NoMethodError
+        # Expected errors from missing config keys or nil config
+        {}
+      end
+
+      private
+
+      # Safely calls merged_config, returning nil on any error.
+      #
+      # @return [Hash, nil] The merged config or nil
+      def safe_merged_config
+        @provider.merged_config
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/skill_bench/services/manifest_finder.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Services
+    # Finds the registry manifest file path.
+    class ManifestFinder
+      # Default path relative to current working directory.
+      DEFAULT_PATH = '../agent-mcp-runtime/registry.json'
+
+      # Finds the registry manifest file.
+      #
+      # @param path [String, nil] Optional custom path to the manifest
+      # @return [String] Absolute path to the registry manifest
+      # @raise [ArgumentError] when the manifest file is not found
+      def self.call(path: nil)
+        new(path: path).call
+      end
+
+      # @param path [String, nil] Optional custom path to the manifest
+      def initialize(path: nil)
+        @path = path
+      end
+
+      # Finds the registry manifest file.
+      #
+      # @return [String] Absolute path to the registry manifest
+      # @raise [ArgumentError] when the manifest file is not found
+      def call
+        manifest_path = @path || File.expand_path(DEFAULT_PATH, Dir.pwd)
+        raise ArgumentError, "Registry manifest not found: #{manifest_path}" unless File.exist?(manifest_path)
+
+        manifest_path
+      end
+    end
+  end
+end

data/lib/skill_bench/services/output_formatter.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Services
+    # Formats agent output for evaluation.
+    class OutputFormatter
+      # Formats agent output for evaluation.
+      #
+      # @param agent_result [Hash] The agent result containing the output
+      # @return [String] The formatted output
+      def self.call(agent_result)
+        new(agent_result).call
+      end
+
+      # @param agent_result [Hash] The agent result containing the output
+      def initialize(agent_result)
+        @agent_result = agent_result
+      end
+
+      # Formats agent output for evaluation.
+      #
+      # @return [String] The formatted output
+      def call
+        @agent_result[:result].to_s
+      end
+    end
+  end
+end

data/lib/skill_bench/services/prompt_builder_service.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require_relative '../execution/context_hydrator'
+require_relative '../execution/source_path_resolver'
+
+module SkillBench
+  module Services
+    # Builds system prompts for baseline and context agent runs.
+    class PromptBuilderService
+      # Builds the baseline system prompt (no skill context).
+      #
+      # @return [String] The baseline system prompt
+      def self.build_baseline
+        new.build_baseline
+      end
+
+      # Builds the context-aware system prompt based on eval metadata.
+      #
+      # For `skill_bundle_xml` context mode, combines SKILL.md with source code
+      # via ContextHydrator. Falls back to SKILL.md-only if source is unavailable.
+      #
+      # @param evaluation [SkillBench::Models::Eval] The eval being run
+      # @param skills [Array<SkillBench::Models::Skill>] Resolved skills
+      # @param skill_context [String] The combined skill context from SKILL.md files
+      # @return [String] The context system prompt
+      def self.build_context(evaluation, skills, skill_context)
+        new.build_context(evaluation, skills, skill_context)
+      end
+
+      # Builds the baseline system prompt (no skill context).
+      #
+      # @return [String] The baseline system prompt
+      def build_baseline
+        <<~PROMPT
+          You are an expert Ruby on Rails developer. Your job is to read the task,
+          modify the codebase using the tools provided to meet the requirements,
+          and then explain what you did.
+        PROMPT
+      end
+
+      # Builds the context-aware system prompt based on eval metadata.
+      #
+      # For `skill_bundle_xml` context mode, combines SKILL.md with source code
+      # via ContextHydrator. Falls back to SKILL.md-only if source is unavailable.
+      #
+      # @param evaluation [SkillBench::Models::Eval] The eval being run
+      # @param _skills [Array<SkillBench::Models::Skill>] Resolved skills (unused in current implementation)
+      # @param skill_context [String] The combined skill context from SKILL.md files
+      # @return [String] The context system prompt
+      def build_context(evaluation, _skills, skill_context)
+        return skill_context unless evaluation.metadata['context_mode'] == 'skill_bundle_xml'
+
+        source_path = resolve_source_path(evaluation)
+        return skill_context unless source_path
+
+        xml_result = Execution::ContextHydrator.call(source_path: source_path, base_path: Pathname.new(Dir.pwd))
+        hydrator_response = xml_result[:response]
+        xml_context = hydrator_response[:context]
+        return skill_context unless xml_result[:success] && !xml_context.empty?
+
+        <<~PROMPT
+          You are an expert Ruby on Rails developer.
+          You have access to a skill file and source code wrapped in <agent_context> tags.
+          Use the skill instructions and the provided source code to solve the task.
+
+          ## Skill Instructions
+          #{skill_context}
+
+          ## Source Code
+          #{xml_context}
+        PROMPT
+      end
+
+      private
+
+      # Resolves the source path for context hydration.
+      #
+      # Tries the eval's `source/` subdirectory first, then falls back to
+      # SourcePathResolver inference.
+      #
+      # @param evaluation [SkillBench::Models::Eval] The eval being run
+      # @return [String, nil] The resolved source path, or nil if not found
+      def resolve_source_path(evaluation)
+        eval_path = evaluation.path
+        eval_source = File.join(eval_path, 'source')
+        return eval_source if Dir.exist?(eval_source)
+
+        sources = SkillBench::Config.skill_sources || {}
+        inferred = Execution::SourcePathResolver.call(
+          eval_folder_path: eval_path.to_s,
+          skill_sources: sources
+        )
+        inferred if inferred && Dir.exist?(inferred)
+      end
+    end
+  end
+end

data/lib/skill_bench/services/provider_resolver.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require_relative '../models/config'
+require_relative '../models/provider'
+
+module SkillBench
+  module Services
+    # Resolves the provider and its configuration.
+    class ProviderResolver
+      # Stand-in provider when no LLM config is available.
+      MOCK_PROVIDER = Struct.new(:name, :runtime, :llm, :merged_config)
+      private_constant :MOCK_PROVIDER
+
+      # Resolves the provider and its configuration.
+      #
+      # @return [Hash] Result with keys:
+      #   - success: Boolean indicating if resolution succeeded
+      #   - provider: The resolved provider instance
+      #   - config: The merged provider config (if successful)
+      #   - error: The error object (if failed)
+      def self.call
+        new.call
+      end
+
+      # Resolves the provider and its configuration.
+      #
+      # @return [Hash] Result with keys:
+      #   - success: Boolean indicating if resolution succeeded
+      #   - provider: The resolved provider instance
+      #   - config: The merged provider config (if successful)
+      #   - error: The error object (if failed)
+      def call
+        provider = resolve_provider
+        config_result = resolve_provider_config(provider)
+
+        if config_result[:success]
+          {
+            success: true,
+            provider: provider,
+            config: config_result[:config]
+          }
+        else
+          {
+            success: false,
+            provider: provider,
+            error: config_result[:error]
+          }
+        end
+      end
+
+      private
+
+      def resolve_provider
+        config = SkillBench::Models::Config.load
+        provider = config.to_provider
+        return provider if provider
+
+        warn 'Config load failed, using mock provider'
+        MOCK_PROVIDER.new('mock', 'mock', 'mock', {})
+      rescue JSON::ParserError, ArgumentError, Errno::ENOENT => e
+        # Config parsing/validation errors or missing config file - fall back to mock
+        warn "Config load failed with error: #{e.message}, using mock provider"
+        MOCK_PROVIDER.new('mock', 'mock', 'mock', {})
+      end
+
+      def resolve_provider_config(provider)
+        { success: true, config: provider.merged_config }
+      rescue ArgumentError => e
+        { success: false, error: e }
+      end
+    end
+  end
+end