ruby-skill-bench 0.1.0

data/lib/skill_bench/evaluation/generator.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'fileutils'
+require_relative '../services/skill_resolver'
+require_relative '../error_logger'
+require_relative '../models/config'
+require_relative '../models/criteria_validator'
+
+module SkillBench
+  module Evaluation
+    # Generates an eval (task.md + criteria.json) from a skill's documentation.
+    class Generator
+      # Prompt template used to generate evals from skill documentation via LLM.
+      GENERATION_PROMPT = <<~PROMPT
+        You are an evaluation designer for a skill-benchmarking tool.
+
+        Given a skill's documentation, create an eval scenario that tests whether an AI agent
+        can apply the skill correctly. Output ONLY a JSON object with this exact structure:
+
+        {
+          "task": "A detailed task description for the agent to perform. Be specific about what the agent should build or do.",
+          "context": "A brief description of what this eval measures.",
+          "dimensions": [
+            { "name": "correctness", "max_score": 30 },
+            { "name": "skill_adherence", "max_score": 25 },
+            { "name": "code_quality", "max_score": 20 },
+            { "name": "test_coverage", "max_score": 15 },
+            { "name": "documentation", "max_score": 10 }
+          ],
+          "pass_threshold": 70,
+          "minimum_delta": 10
+        }
+
+        Rules:
+        - dimension max_scores MUST sum to exactly 100
+        - pass_threshold should be between 60 and 80
+        - minimum_delta should be between 5 and 15
+        - task should be specific enough that an agent can attempt it in under 5 minutes
+        - the eval should test whether the agent follows the patterns from the skill
+
+        Skill documentation:
+      PROMPT
+
+      # @param skill_name [String] Name of the skill to base the eval on.
+      # @param eval_name [String] Name for the new eval directory.
+      def initialize(skill_name:, eval_name:)
+        @skill_name = skill_name
+        @eval_name = eval_name
+      end
+
+      # Generates the eval files.
+      #
+      # @return [Hash] Service response.
+      def call
+        sanitized = sanitize_eval_name(eval_name)
+        return invalid_name_result unless sanitized
+
+        skill = resolve_skill
+        return skill_not_found_result unless skill
+
+        skill_content = read_skill_content(skill.path)
+        generated = generate_eval(skill_content)
+        return generated unless generated[:success]
+
+        write_eval_files(sanitized, generated[:response][:data])
+
+        criteria_path = File.join('evals', sanitized, 'criteria.json')
+        validation = SkillBench::Models::CriteriaValidator.call(path: criteria_path)
+        unless validation[:success]
+          FileUtils.rm_rf(File.join('evals', sanitized))
+          return validation
+        end
+
+        { success: true, response: { eval_path: "evals/#{sanitized}" } }
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'Evaluation::Generator Error')
+        { success: false, response: { error: { message: e.message } } }
+      end
+
+      private
+
+      attr_reader :skill_name, :eval_name
+
+      def sanitize_eval_name(name)
+        stripped = name&.strip
+        return nil if stripped.nil? || stripped.empty?
+        return nil if stripped == '.'
+        return nil if stripped.include?('..') || stripped.start_with?('/') || stripped =~ %r{[\\/:]}
+
+        stripped
+      end
+
+      def invalid_name_result
+        { success: false, response: { error: { message: "Invalid eval name: #{eval_name}" } } }
+      end
+
+      def resolve_skill
+        Services::SkillResolver.call(skill_name)
+      rescue ArgumentError
+        nil
+      end
+
+      def skill_not_found_result
+        { success: false, response: { error: { message: "Skill not found: #{skill_name}" } } }
+      end
+
+      def read_skill_content(skill_path)
+        skill_md = File.join(skill_path, 'SKILL.md')
+        File.exist?(skill_md) ? File.read(skill_md) : ''
+      end
+
+      def generate_eval(skill_content)
+        prompt = GENERATION_PROMPT + "\n\n#{skill_content}"
+
+        provider = load_provider
+        return mock_generate if provider.nil? || provider.name == 'mock'
+
+        client_class = SkillBench::Clients::ProviderRegistry.for(provider.runtime.to_sym)
+        response = client_class.call(
+          system_prompt: '',
+          messages: [{ role: 'user', content: prompt }],
+          model: provider.llm,
+          **provider.merged_config
+        )
+
+        return { success: false, response: { error: { message: 'LLM generation failed' } } } unless response[:success]
+
+        parse_generated_json(response[:result])
+      end
+
+      def load_provider
+        config = SkillBench::Models::Config.load
+        config.to_provider
+      rescue Errno::ENOENT
+        nil
+      end
+
+      def mock_generate
+        parse_generated_json(<<~JSON)
+          {
+            "task": "Apply the skill patterns to solve a representative task.",
+            "context": "Evaluate skill application",
+            "dimensions": [
+              { "name": "correctness", "max_score": 30 },
+              { "name": "skill_adherence", "max_score": 25 },
+              { "name": "code_quality", "max_score": 20 },
+              { "name": "test_coverage", "max_score": 15 },
+              { "name": "documentation", "max_score": 10 }
+            ],
+            "pass_threshold": 70,
+            "minimum_delta": 10
+          }
+        JSON
+      end
+
+      def parse_generated_json(json_text)
+        data = JSON.parse(json_text)
+        { success: true, response: { data: data } }
+      rescue JSON::ParserError => e
+        { success: false, response: { error: { message: "Failed to parse generated eval: #{e.message}" } } }
+      end
+
+      def write_eval_files(sanitized_name, data)
+        eval_dir = File.join('evals', sanitized_name)
+        FileUtils.mkdir_p(eval_dir)
+
+        File.write(File.join(eval_dir, 'task.md'), data['task'] || data[:task] || '')
+        File.write(File.join(eval_dir, 'criteria.json'), JSON.pretty_generate(build_criteria_hash(data)))
+      end
+
+      def build_criteria_hash(data)
+        {
+          context: data.fetch('context', data[:context] || ''),
+          dimensions: data.fetch('dimensions', data[:dimensions] || []),
+          pass_threshold: extract_numeric(data, 'pass_threshold', 70),
+          minimum_delta: extract_numeric(data, 'minimum_delta', 10)
+        }
+      end
+
+      def extract_numeric(data, key, default)
+        return data[key] if data.key?(key)
+
+        sym = key.to_sym
+        return data[sym] if data.key?(sym)
+
+        default
+      end
+    end
+  end
+end

data/lib/skill_bench/evaluation/runner.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Evaluation
+    # Orchestrates the evaluation pipeline.
+    #
+    # Coordinates blind judging of baseline and context agent outputs,
+    # then computes deltas and determines the final verdict.
+    class Runner
+      # Runs the evaluation pipeline.
+      #
+      # @param task [String] The task description.
+      # @param criteria [SkillBench::Criteria] The eval criteria.
+      # @param skill_context [String] The skill context XML.
+      # @param baseline_output [String] The baseline agent output.
+      # @param context_output [String] The context agent output.
+      # @param judge_params [Hash] Provider config passed to the Judge as client_params (api_key, model, provider).
+      # @return [Hash] Service response with report or error.
+      def self.call(task:, criteria:, skill_context:, baseline_output:, context_output:, judge_params: {})
+        new(task:, criteria:, skill_context:, baseline_output:, context_output:, judge_params:).call
+      end
+
+      # @param task [String] The task description.
+      # @param criteria [SkillBench::Criteria] The eval criteria.
+      # @param skill_context [String] The skill context XML.
+      # @param baseline_output [String] The baseline agent output.
+      # @param context_output [String] The context agent output.
+      # @param judge_params [Hash] Provider config passed to the Judge as client_params.
+      def initialize(task:, criteria:, skill_context:, baseline_output:, context_output:, judge_params: {})
+        @task = task
+        @criteria = criteria
+        @skill_context = skill_context
+        @baseline_output = baseline_output
+        @context_output = context_output
+        @judge_params = judge_params.is_a?(Hash) ? judge_params : {}
+      end
+
+      # Orchestrates judging and delta computation.
+      #
+      # @return [Hash] Service response with report or error.
+      def call
+        baseline_judge = judge_run(baseline_output, nil)
+        return baseline_judge unless baseline_judge[:success]
+
+        context_judge = judge_run(context_output, skill_context)
+        return context_judge unless context_judge[:success]
+
+        compute_deltas(baseline_judge, context_judge)
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'Evaluation::Runner Error')
+        { success: false, response: { error: { message: e.message } } }
+      end
+
+      private
+
+      attr_reader :task, :criteria, :skill_context, :baseline_output, :context_output, :judge_params
+
+      def judge_run(output, context)
+        prompt_result = Judge::Prompt.call(
+          task: task,
+          criteria: criteria,
+          skill_context: context,
+          agent_output: output
+        )
+        return prompt_result unless prompt_result[:success]
+
+        Judge::Judge.call(prompt: prompt_result[:response][:prompt], client_params: judge_params)
+      end
+
+      def compute_deltas(baseline_judge, context_judge)
+        baseline_dims = baseline_judge[:response][:judge_response].dimensions
+        context_dims = context_judge[:response][:judge_response].dimensions
+
+        delta_result = DeltaReport.call(baseline: baseline_dims, context: context_dims, criteria: criteria)
+        return delta_result unless delta_result[:success]
+
+        { success: true, response: { report: delta_result[:response][:delta_report] } }
+      end
+    end
+  end
+end

data/lib/skill_bench/evaluation.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module SkillBench
+  # Namespace for the evaluation orchestration subsystem.
+  #
+  # Coordinates evaluation workflows across multiple tasks,
+  # including blind judging and delta computation.
+  module Evaluation
+  end
+end

data/lib/skill_bench/execution/context_hydrator.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require 'cgi'
+
+module SkillBench
+  module Execution
+    # Responsible for loading source context files from a given path
+    # and wrapping them in XML tags for injection into the LLM system prompt.
+    class ContextHydrator
+      # Error message returned when context hydration fails.
+      HYDRATION_FAILED = 'Failed to hydrate context from source path'
+      # File extensions considered for context hydration.
+      TEXT_EXTENSIONS = %w[.md .rb .json .yml .yaml .txt].freeze
+      # Maximum file size (in bytes) for files included in context hydration.
+      MAX_FILE_SIZE = 50_000
+
+      # Loads and formats source context files.
+      #
+      # @param params [Hash] The configuration for context hydration.
+      # @option params [String] :source_path The path to the source directory containing readable files.
+      # @option params [String] :skill_path Deprecated alias for `:source_path`.
+      # @option params [Pathname, String] :base_path (optional) The base path to resolve the source directory against.
+      # @return [Hash] A result hash with :success, and :response containing the XML formatted context.
+      # @raise [TypeError] when the provided source or base path cannot be converted into a pathname.
+      def self.call(params)
+        new(**params).call
+      end
+
+      # @param source_path [String] The path to the source directory containing readable files.
+      # @param skill_path [String] Deprecated alias for source_path.
+      # @param base_path [Pathname, String] The base path to resolve the source directory against.
+      # @return [void]
+      # @raise [TypeError] when the provided source or base path cannot be converted into a pathname.
+      def initialize(source_path: nil, skill_path: nil, base_path: nil)
+        @source_path = source_path || skill_path
+        @base_path = base_path || Pathname.new(Dir.pwd)
+      end
+
+      # Performs the hydration process.
+      #
+      # @return [Hash] The standardized result hash indicating success or failure.
+      def call
+        return missing_path_result unless @source_path
+
+        full_path = @base_path.join(@source_path).expand_path
+        base_expanded = @base_path.expand_path
+
+        return missing_path_result unless full_path.to_path.start_with?(base_expanded.to_path)
+        return missing_path_result unless full_path.exist? && full_path.directory?
+
+        context_files = collect_context_files(full_path)
+        xml_context = build_xml(context_files)
+
+        { success: true, response: { context: xml_context } }
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'Hydration Error')
+        { success: false, response: { error: { message: e.message } } }
+      end
+
+      private
+
+      def missing_path_result
+        { success: false, response: { error: { message: "Source path #{@source_path} does not exist or is not a directory" } } }
+      end
+
+      def collect_context_files(full_path)
+        pattern = full_path.join("*{#{TEXT_EXTENSIONS.join(',')}}").to_s
+        Dir.glob(pattern).reject { |f| File.symlink?(f) }
+                         .select { |f| File.size(f) <= MAX_FILE_SIZE }
+                         .sort
+      end
+
+      # Builds the XML structure wrapping the contents of the context files.
+      #
+      # @param context_files [Array<String>] List of absolute paths to context files.
+      # @return [String] The combined XML representation of the file contents.
+      def build_xml(context_files)
+        return '' if context_files.empty?
+
+        xml = ['<agent_context>']
+
+        context_files.each do |file_path|
+          relative_path = Pathname.new(file_path).relative_path_from(@base_path).to_s
+          content = File.read(file_path)
+
+          xml << "  <file path=\"#{CGI.escapeHTML(relative_path)}\">"
+          xml << CGI.escapeHTML(content).gsub(/^/, '    ')
+          xml << '  </file>'
+        end
+
+        xml << '</agent_context>'
+        xml.join("\n")
+      end
+    end
+  end
+end

data/lib/skill_bench/execution/sandbox.rb

@@ -0,0 +1,174 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'tmpdir'
+require 'open3'
+
+module SkillBench
+  module Execution
+    # Manages isolated sandbox environments for running agent evaluations.
+    # Handles copying files, initializing git, and capturing diffs.
+    # Now supports Docker container isolation for secure command execution.
+    class Sandbox
+      attr_reader :path, :container_id
+
+      # Runs a block of code within a temporary, isolated sandbox directory.
+      # The sandbox is initialized as a git repository and optionally wrapped in a Docker container.
+      #
+      # @param source_dir [String, Pathname] The directory to copy into the sandbox.
+      # @yieldparam sandbox [SkillBench::Execution::Sandbox] The sandbox instance.
+      # @return [Object] The result of the yielded block.
+      # @raise [SystemCallError] when file operations or directory creation fails.
+      # @raise [RuntimeError] when Docker commands fail.
+      def self.run(source_dir, &)
+        new(source_dir).run(&)
+      end
+
+      # @param source_dir [String, Pathname] The directory to copy into the sandbox.
+      def initialize(source_dir)
+        @source_dir = source_dir
+        @path = nil
+        @container_id = nil
+      end
+
+      # Executes the sandbox environment setup and yields the sandbox instance.
+      #
+      # @yieldparam sandbox [SkillBench::Execution::Sandbox] The sandbox instance.
+      # @return [Object] The result of the yielded block.
+      # @raise [SystemCallError] when file operations or directory creation fails.
+      # @raise [RuntimeError] when Docker commands fail.
+      def run
+        Dir.mktmpdir('evaluator_sandbox_') do |sandbox_dir|
+          @path = sandbox_dir
+          copy_source_files(sandbox_dir)
+
+          setup_git
+
+          start_container if docker_available?
+          begin
+            yield self
+          ensure
+            stop_container
+          end
+        end
+      end
+
+      # Captures the git diff of changes made within the sandbox.
+      #
+      # @param sandbox_dir [String] The path to the sandbox directory.
+      # @return [String] The git diff, or a message indicating no changes.
+      # @raise [SystemCallError] when git commands fail.
+      def self.capture_diff(sandbox_dir)
+        sandbox_path = File.realpath(sandbox_dir)
+        tmp_prefix = File.realpath(Dir.tmpdir) + File::SEPARATOR
+        raise "Sandbox directory #{sandbox_dir} is outside temp directory" unless sandbox_path.start_with?(tmp_prefix)
+
+        return 'No code changes made.' unless File.directory?(File.join(sandbox_path, '.git'))
+
+        raise "Failed to stage changes in #{sandbox_path}" unless system('git', 'add', '.', chdir: sandbox_path)
+
+        diff, status = Open3.capture2('git', 'diff', '--cached', chdir: sandbox_path)
+        raise "Failed to capture diff in #{sandbox_path}" unless status.success?
+
+        diff.strip.empty? ? 'No code changes made.' : diff
+      end
+
+      private
+
+      def setup_git
+        cmds = [
+          ['git', 'init', '--quiet'],
+          ['git', 'config', 'user.email', 'evaluator@tessl.io'],
+          ['git', 'config', 'user.name', 'Evaluator Sandbox'],
+          ['git', 'add', '.'],
+          ['git', 'commit', '--quiet', '-m', 'Initial commit']
+        ]
+
+        cmds.each do |argv|
+          raise "Git command failed: #{argv.join(' ')}" unless system(*argv, chdir: @path)
+        end
+      end
+
+      # Copies source files into the sandbox, including dotfiles.
+      # Validates symlinks to prevent path traversal.
+      #
+      # @param sandbox_dir [String] The destination sandbox directory.
+      # @raise [RuntimeError] when a symlink points outside the source directory.
+      def copy_source_files(sandbox_dir)
+        source_real = File.realpath(@source_dir)
+        copy_tree(@source_dir, sandbox_dir, source_real)
+      end
+
+      def copy_tree(src_dir, dst_dir, source_real)
+        Dir.entries(src_dir).each do |entry|
+          next if %w[. ..].include?(entry)
+
+          src = File.join(src_dir, entry)
+          dst = File.join(dst_dir, entry)
+
+          if File.symlink?(src)
+            real = File.realpath(src)
+            raise "Symlink #{entry} points outside source directory" unless real.start_with?("#{source_real}/")
+
+            copy_item(real, dst, source_real)
+          elsif File.directory?(src)
+            copy_item(src, dst, source_real)
+          else
+            FileUtils.cp(src, dst)
+          end
+        end
+      end
+
+      def copy_item(src, dst, source_real)
+        FileUtils.mkdir_p(dst)
+        if File.directory?(src)
+          copy_tree(src, dst, source_real)
+        else
+          FileUtils.cp(src, dst)
+        end
+      end
+
+      # Checks if Docker is available and the sandbox Dockerfile exists.
+      #
+      # @return [Boolean] true if Docker is available, false otherwise.
+      def docker_available?
+        docker_dir = File.expand_path('docker', __dir__)
+        return false unless File.directory?(docker_dir)
+
+        _stdout, _stderr, status = Open3.capture3('docker', 'info')
+        status.success?
+      rescue Errno::ENOENT
+        false
+      end
+
+      # Starts a Docker container for isolated command execution.
+      # Builds the image only if it does not already exist.
+      #
+      # @raise [RuntimeError] when the Docker image cannot be built or the container fails to start.
+      def start_container
+        image_name = 'evaluator-sandbox'
+        docker_dir = File.expand_path('docker', __dir__)
+
+        # Build image (Docker layer cache handles no-op builds)
+        raise "Failed to build Docker image #{image_name}" unless system('docker', 'build', '-t', image_name, docker_dir, '--quiet')
+
+        # Start a detached container mounting the sandbox dir to /sandbox
+        stdout, stderr, status = Open3.capture3(
+          'docker', 'run', '-d', '--rm', '-v', "#{@path}:/sandbox", image_name
+        )
+
+        raise "Failed to start Docker container: #{stderr}" unless status.success?
+
+        @container_id = stdout.strip
+      end
+
+      def stop_container
+        return unless @container_id
+
+        # Stop and remove the container (it's --rm so stopping also removes it)
+        # We don't fail-fast on stop to avoid swallowing the original error if this is in an ensure block
+        system('docker', 'stop', @container_id, out: File::NULL, err: File::NULL)
+      end
+    end
+  end
+end

data/lib/skill_bench/execution/source_path_resolver.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Execution
+    # Resolves the source skill or workflow path for a given evaluation target.
+    class SourcePathResolver
+      # Resolves the source path using either an explicit override or the eval directory convention.
+      #
+      # @param eval_folder_path [String] Relative path to the eval directory.
+      # @param skill_path [String, nil] Optional explicit override for the source directory.
+      # @return [String, nil] The resolved source path relative to the evaluator repo root, or nil if unmappable.
+      # @example Infer a skill source path (NEW format):
+      #   SkillBench::Execution::SourcePathResolver.call(
+      #     eval_folder_path: 'evals/skills/rails-code-review/review-order'
+      #   )
+      #   # => "skills/rails-code-review"
+      # @example Infer a skill source path (OLD format, returns category):
+      #   SkillBench::Execution::SourcePathResolver.call(
+      #     eval_folder_path: 'evals/skills/code-quality/rails-code-review/review-order'
+      #   )
+      #   # => "skills/code-quality/rails-code-review"
+      def self.call(eval_folder_path:, skill_path: nil)
+        return skill_path if skill_path && !skill_path.empty?
+
+        segments = eval_folder_path.to_s.split('/').reject(&:empty?)
+
+        resolve_skills_path(segments) || resolve_workflows_path(segments)
+      end
+
+      private_class_method def self.resolve_skills_path(segments)
+        return nil unless (index = segments.rindex('skills'))
+
+        remaining = segments[(index + 1)..]
+        resolve_old_format_skills(remaining) || resolve_new_format_skills(remaining)
+      end
+
+      private_class_method def self.resolve_old_format_skills(remaining)
+        return nil unless remaining.size >= 3
+
+        category = remaining[0]
+        skill_name = remaining[1]
+        "skills/#{category}/#{skill_name}"
+      end
+
+      private_class_method def self.resolve_new_format_skills(remaining)
+        return nil unless remaining.size >= 1
+
+        skill_name = remaining[0]
+        "skills/#{skill_name}"
+      end
+
+      private_class_method def self.resolve_workflows_path(segments)
+        return nil unless (index = segments.rindex('workflows'))
+
+        workflow_name = segments[index + 1]
+        "workflows/#{workflow_name}" if workflow_name
+      end
+    end
+  end
+end

data/lib/skill_bench/execution.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module SkillBench
+  # Namespace for the execution environment subsystem.
+  #
+  # Provides isolated execution environments for agent evaluation,
+  # including sandbox management and context hydration.
+  module Execution
+  end
+end