ruby-skill-bench 0.1.0

data/lib/skill_bench/package_verifier.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require 'rubygems/package'
+require_relative 'error_logger'
+
+module SkillBench
+  # Verifies that a built gem package includes the files required for release.
+  class PackageVerifier
+    # Raised when a built gem package is missing required files.
+    class Error < StandardError; end
+
+    # Files that must be present for a usable evaluator gem package.
+    REQUIRED_FILES = %w[
+      README.md
+      LICENSE
+      bin/skill-bench
+      docs/architecture.md
+      docs/testing-guide.md
+      lib/skill_bench.rb
+      lib/skill_bench/config/applier.rb
+      lib/skill_bench/config/defaults.rb
+      lib/skill_bench/config/env_overrides.rb
+      lib/skill_bench/config/facade_readers.rb
+      lib/skill_bench/config/facade_writers.rb
+      lib/skill_bench/config/json_loader.rb
+      lib/skill_bench/config/store.rb
+      lib/skill_bench/package_verifier.rb
+      lib/skill_bench/source_path_resolver.rb
+      lib/skill_bench/runner.rb
+    ].freeze
+
+    # Verifies that a gem package includes required release files.
+    #
+    # @param package_path [String] path to the built `.gem` file
+    # @param required_files [Array<String>] files that must be present in the gemspec payload
+    # @return [Hash] result envelope with package verification details
+    def self.call(package_path:, required_files: REQUIRED_FILES)
+      new(package_path:, required_files:).call
+    end
+
+    # Initializes the verifier.
+    #
+    # @param package_path [String] path to the built `.gem` file
+    # @param required_files [Array<String>] files that must be present in the gemspec payload
+    # @return [PackageVerifier] a verifier instance
+    def initialize(package_path:, required_files: REQUIRED_FILES)
+      @package_path = package_path
+      @required_files = required_files
+    end
+
+    # Verifies that the configured package contains all required files.
+    #
+    # @return [Hash] result envelope with package verification details
+    def call
+      files = packaged_files
+      missing = @required_files - files
+      return failure("Missing packaged files: #{missing.join(', ')}") if missing.any?
+
+      { success: true, response: { missing_files: [], packaged_files: files } }
+    rescue StandardError => e
+      SkillBench::ErrorLogger.log_error(e, 'PackageVerifier Error')
+      failure(e.message)
+    end
+
+    private
+
+    def packaged_files
+      Gem::Package.new(@package_path).spec.files
+    end
+
+    def failure(message)
+      {
+        success: false,
+        response: {
+          error: { message: message }
+        }
+      }
+    end
+  end
+end

data/lib/skill_bench/rails/skill_templates.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'active_support/inflector'
+
+module SkillBench
+  module Rails
+    # Generates Rails-specific skill templates
+    class SkillTemplates
+      # Generate a service object template
+      # @param name [String] Service name (e.g., 'my_service' or 'my-service')
+      # @return [String] Service object Ruby class
+      def self.service_object(name)
+        class_name = name.split(/[-_]/).map(&:capitalize).join
+        <<~RUBY
+          # frozen_string_literal: true
+
+          module SkillBench
+            module Skills
+              class #{class_name}
+                # Initialize with required parameters
+                # @param args [Hash] Keyword arguments for the service
+                def initialize(**args)
+                  # Set instance variables from args
+                end
+
+                # Execute the service
+                # @return [Hash] Result with :success and :response keys
+                def call
+                  # Implement service logic here
+                  { success: true, response: { message: 'Not implemented' } }
+                rescue StandardError => e
+                  Rails.logger.error(e.message)
+                  Rails.logger.error(e.backtrace.first(5).join("\n"))
+                  { success: false, response: { error: { message: e.message } } }
+                end
+              end
+            end
+          end
+        RUBY
+      end
+
+      # Generate a concern template
+      # @param name [String] Concern name (e.g., 'my_concern')
+      # @return [String] Concern module
+      def self.concern(name)
+        module_name = name.camelize
+        <<~RUBY
+          # frozen_string_literal: true
+
+          module #{module_name}
+            extend ActiveSupport::Concern
+
+            included do
+              # Add class methods, associations, validations here
+            end
+
+            class_methods do
+              # Add class methods here
+            end
+
+            # Add instance methods here
+          end
+        RUBY
+      end
+
+      # Generate an ActiveRecord model template
+      # @param name [String] Model name (e.g., 'my_model')
+      # @return [String] ActiveRecord model class
+      def self.active_record_model(name)
+        class_name = name.camelize
+        <<~RUBY
+          # frozen_string_literal: true
+
+          class #{class_name} < ApplicationRecord
+            # Validations
+            validates :name, presence: true
+
+            # Associations
+            # belongs_to :user
+            # has_many :items
+
+            # Scopes
+            # scope :active, -> { where(active: true) }
+
+            # Instance methods
+            # def some_method
+            #   ...
+            # end
+
+            # Class methods
+            # def self.some_class_method
+            #   ...
+            # end
+          end
+        RUBY
+      end
+    end
+  end
+end

data/lib/skill_bench/runner.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require 'parallel'
+require_relative 'task/evaluator'
+require_relative 'error_logger'
+
+module SkillBench
+  # Orchestrates the entire evaluation process.
+  # Compares how an AI coding agent performs with and without contextual skills.
+  # @deprecated Use {SkillBench::Services::RunnerService} instead.
+  class Runner
+    # Initiates a full evaluation run.
+    #
+    # @param params [Hash] The configuration for the evaluation.
+    # @option params [String] :eval_folder_path The path to the evaluation directory containing task and criteria.
+    # @option params [String] :skill_path Optional override for the source directory being tested.
+    # @option params [String, Pathname] :base_path (optional) The base path for relative file resolution.
+    # @option params [Hash] :client_params (optional) Parameters to pass to the LLM client.
+    # @return [Hash] A result hash with :success and :response payload containing the judge scores and diffs.
+    # @raise [ArgumentError] If the eval path does not match a supported source-path convention.
+    def self.call(params)
+      new(params).call
+    end
+
+    # @param params [Hash] The configuration for the evaluation.
+    def initialize(params)
+      @eval_folder_path = params[:eval_folder_path]
+      @skill_path = params[:skill_path]
+      @base_path = params[:base_path] || Pathname.new(Dir.pwd)
+      @client_params = params[:client_params] || {}
+    end
+
+    # Executes the baseline and context-hydrated evaluations, then scores them.
+    #
+    # @return [Hash] The final evaluation result.
+    def call
+      full_path = @base_path.join(@eval_folder_path)
+
+      return { success: false, response: { error: { message: "Evaluation path #{full_path} does not exist" } } } unless full_path.exist?
+
+      task_dirs = self.class.discover_task_dirs(full_path)
+      if task_dirs.empty?
+        return { success: false,
+                 response: { error: { message: "No task.md found in #{full_path} or its subdirectories" } } }
+      end
+
+      results = Parallel.map(task_dirs, in_threads: 4) do |task_dir|
+        task_result = Task::Evaluator.call(
+          full_eval_path: task_dir,
+          base_path: @base_path,
+          skill_path: @skill_path,
+          client_params: @client_params
+        )
+        # Normalize to uniform envelope
+        if task_result.key?(:success)
+          task_result
+        else
+          { success: true, response: task_result }
+        end
+      end
+
+      overall_success = results.all? { |task_result| task_result[:success] }
+
+      {
+        success: overall_success,
+        response: {
+          source_path: @skill_path || 'multiple (batch run)',
+          tasks: results
+        }
+      }
+    rescue StandardError => e
+      SkillBench::ErrorLogger.log_error(e, 'Runner Error')
+      { success: false, response: { error: { message: e.message } } }
+    end
+
+    # Finds all directories containing a task.md file starting from the root_path.
+    #
+    # @param root_path [Pathname] The root directory to search.
+    # @return [Array<Pathname>] A list of task directory paths.
+    def self.discover_task_dirs(root_path)
+      if File.exist?(root_path.join('task.md'))
+        [root_path]
+      else
+        Dir.glob(root_path.join('**/task.md')).map { |f| Pathname.new(f).parent }.uniq.sort
+      end
+    end
+  end
+end

data/lib/skill_bench/services/delta_table_formatter.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require_relative 'formatting_helpers'
+
+module SkillBench
+  module Services
+    # Formats the dimension scoring table, totals, trend, and verdict for a DeltaReport.
+    class DeltaTableFormatter
+      extend FormattingHelpers
+
+      # Formats the delta report scoring section.
+      #
+      # @param report [SkillBench::DeltaReport] The delta report.
+      # @param result [Hash, nil] Eval result envelope (used for trend data).
+      # @return [String] Formatted table, totals, trend, and verdict.
+      def self.format(report, result = nil)
+        lines = [
+          '  DIMENSION                BASELINE   CONTEXT    DELTA',
+          '  ──────────────────────── ───────── ───────── ───────'
+        ]
+
+        report.deltas.each do |name, delta|
+          lines << format_dimension_row(name, delta, report)
+        end
+
+        lines << '  ──────────────────────── ───────── ───────── ───────'
+        lines << format_total_row(report)
+        lines << ''
+        trend = result[:trend] if result
+        lines << format_trend(trend) if trend
+
+        status = report.verdict ? 'PASS' : 'FAIL'
+        criteria = report.criteria
+        threshold = criteria.pass_threshold
+        delta_threshold = criteria.minimum_delta
+        lines << "  VERDICT: #{status} (threshold: #{threshold}, minimum delta: #{delta_threshold})"
+        lines << ('═' * 55)
+
+        lines.join("\n")
+      end
+
+      private_class_method def self.format_dimension_row(name, delta, report)
+        dim = report.criteria.dimensions.find { |d| d.name == name }
+        max_score = dim&.max_score || ''
+        humanized = humanize(name)
+        label = dim ? "#{humanized} (#{max_score})" : humanized
+        baseline_score = report.baseline_scores[name]
+        context_score = report.context_scores[name]
+        Kernel.format('  %<label>-24s %<baseline>9s %<context>9s %<delta>7s',
+                      label: label, baseline: baseline_score, context: context_score,
+                      delta: delta_str(delta))
+      end
+
+      private_class_method def self.format_total_row(report)
+        Kernel.format('  %<label>-24s %<baseline>9s %<context>9s %<delta>7s',
+                      label: 'TOTAL', baseline: "#{report.baseline_total}/100",
+                      context: "#{report.context_total}/100",
+                      delta: delta_str(report.deltas.values.sum))
+      end
+
+      private_class_method def self.format_trend(trend)
+        return nil unless trend
+
+        baseline_icon = trend_icon(trend[:baseline_trend])
+        context_icon = trend_icon(trend[:context_trend])
+        baseline_delta = trend[:baseline_delta]
+        context_delta = trend[:context_delta]
+        "  TREND: baseline #{baseline_icon} (#{delta_str(baseline_delta)}), context #{context_icon} (#{delta_str(context_delta)})"
+      end
+    end
+  end
+end

data/lib/skill_bench/services/feedback_generator.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require_relative 'formatting_helpers'
+
+module SkillBench
+  module Services
+    # Categorizes dimension scores into "what went well", "what went wrong",
+    # and actionable advice based on judge reasoning.
+    class FeedbackGenerator
+      extend FormattingHelpers
+
+      # Generates feedback sections from a DeltaReport.
+      #
+      # @param report [SkillBench::DeltaReport] The delta report.
+      # @return [Hash] Standardized response hash:
+      #   - { success: true, response: { output: String } }
+      def self.call(report)
+        output = generate_feedback(report)
+        { success: true, response: { output: output } }
+      end
+
+      private_class_method def self.generate_feedback(report)
+        return '' unless feedback_applicable?(report)
+
+        context_dims = report.context_dimensions || {}
+        baseline_dims = report.baseline_dimensions || {}
+        well, wrong, advice = categorize_dimensions(context_dims, baseline_dims, report)
+
+        assemble_feedback_lines(well, wrong, advice)
+      end
+
+      private_class_method def self.feedback_applicable?(report)
+        return false unless report.respond_to?(:baseline_dimensions) && report.respond_to?(:context_dimensions)
+
+        context_dims = report.context_dimensions || {}
+        baseline_dims = report.baseline_dimensions || {}
+        context_dims.any? { |name, dim| baseline_dims[name] && dim }
+      end
+
+      private_class_method def self.categorize_dimensions(context_dims, baseline_dims, report)
+        well = []
+        wrong = []
+        advice = []
+
+        context_dims.each do |name, dim|
+          baseline_dim = baseline_dims[name]
+          next unless baseline_dim && dim
+
+          cat = categorize_dimension(name, dim, baseline_dim, report)
+          well.concat(cat[:well])
+          wrong.concat(cat[:wrong])
+          advice.concat(cat[:advice])
+        end
+
+        [well, wrong, advice]
+      end
+
+      private_class_method def self.categorize_dimension(name, dim, baseline_dim, report)
+        values = extract_values(dim, baseline_dim)
+        score = values[:score]
+        max_score = values[:max_score]
+        baseline_score = values[:baseline_score]
+        reasoning = values[:reasoning]
+
+        pct = compute_percentage(score, max_score)
+        dim_obj = report.criteria.dimensions.find { |d| d.name == name }
+        humanized = humanize(name)
+        label = "#{humanized} (#{score}/#{max_score}, baseline: #{baseline_score}/#{max_score})"
+
+        build_categorization(pct, label, reasoning, humanized, dim_obj)
+      end
+
+      private_class_method def self.extract_values(dim, baseline_dim)
+        {
+          score: dim[:score] || dim['score'] || 0,
+          max_score: dim[:max_score] || dim['max_score'] || 1,
+          reasoning: dim[:reasoning] || dim['reasoning'] || '',
+          baseline_score: baseline_dim[:score] || baseline_dim['score'] || 0
+        }
+      end
+
+      private_class_method def self.compute_percentage(score, max_score)
+        max_score.positive? ? (score.to_f / max_score * 100).round : 0
+      end
+
+      private_class_method def self.build_categorization(pct, label, reasoning, humanized, dim_obj)
+        well = []
+        wrong = []
+        advice = []
+        has_reasoning = !reasoning.empty?
+
+        if pct >= 80
+          well << "  #{label}"
+          well << "    #{reasoning}" if has_reasoning
+        else
+          wrong << "  #{label}"
+          wrong << "    #{reasoning}" if has_reasoning
+          dim_advice = dim_obj&.description.to_s
+          advice << "  #{humanized}: #{dim_advice}" unless dim_advice.empty?
+        end
+
+        { well: well, wrong: wrong, advice: advice }
+      end
+
+      private_class_method def self.assemble_feedback_lines(well, wrong, advice)
+        lines = []
+        append_section(lines, 'WHAT WENT WELL', well)
+        append_section(lines, 'WHAT WENT WRONG', wrong)
+        append_section(lines, 'ADVICE', advice)
+        lines.join("\n")
+      end
+
+      private_class_method def self.append_section(lines, title, items)
+        return if items.empty?
+
+        lines << ''
+        lines << "  === #{title} ==="
+        lines.concat(items)
+      end
+    end
+  end
+end

data/lib/skill_bench/services/formatting_helpers.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Services
+    # Shared string-formatting utilities used across output formatters.
+    module FormattingHelpers
+      module_function
+
+      # Converts a snake_case name to Title Case.
+      #
+      # @param name [String, Symbol] The dimension name.
+      # @return [String] Human-readable name.
+      def humanize(name)
+        name.to_s.split('_').map(&:capitalize).join(' ')
+      end
+
+      # Formats a numeric delta with a +/- sign.
+      #
+      # @param delta [Numeric] The delta value.
+      # @return [String] Formatted delta string.
+      def delta_str(delta)
+        delta >= 0 ? "+#{delta}" : delta.to_s
+      end
+
+      # Truncates a string to a maximum length with ellipsis.
+      #
+      # @param text [String] The text to truncate.
+      # @param max_length [Integer] Maximum length.
+      # @return [String] Truncated text.
+      def truncate(text, max_length)
+        return text if text.length <= max_length
+
+        "#{text[0...max_length]}..."
+      end
+
+      # Returns the Unicode arrow icon for a trend direction.
+      #
+      # @param direction [Symbol] :improved, :regressed, or :unchanged.
+      # @return [String] Arrow icon.
+      def trend_icon(direction)
+        { improved: '↑', regressed: '↓', unchanged: '→' }.fetch(direction, '?')
+      end
+    end
+  end
+end

data/lib/skill_bench/services/iteration_formatter.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative 'formatting_helpers'
+
+module SkillBench
+  module Services
+    # Formats ReAct loop iteration timelines for human-readable output.
+    class IterationFormatter
+      extend FormattingHelpers
+
+      # Formats an iteration timeline section.
+      #
+      # @param title [String] Section title.
+      # @param iterations [Array<Hash>] Iteration metadata with keys :step_number,
+      #   :thought, :tools_used, :observation_summary.
+      # @return [String] Formatted section.
+      def self.format(title, iterations)
+        lines = ["  === #{title} ==="]
+        iterations.each do |iter|
+          tools = iter[:tools_used] || []
+          tool_str = tools.empty? ? '' : " → Tool: #{tools.join(', ')}"
+          observation = iter[:observation_summary].to_s
+          observation_str = observation.empty? ? '' : " → Observation: #{truncate(observation, 60)}"
+          lines << "  Step #{iter[:step_number]}: #{iter[:thought]}#{tool_str}#{observation_str}"
+        end
+        lines.join("\n")
+      end
+    end
+  end
+end

data/lib/skill_bench/services/json_formatter.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module SkillBench
+  module Services
+    # Formats evaluation results as JSON.
+    class JsonFormatter
+      # Format result as JSON.
+      #
+      # @param result [Hash] Eval result.
+      # @return [String] JSON-formatted string.
+      def self.format(result)
+        JSON.pretty_generate(result)
+      end
+    end
+  end
+end

data/lib/skill_bench/services/judge_score_parser_service.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module SkillBench
+  module Services
+    # Service object for parsing judge score responses from evaluation results.
+    # Handles JSON strings with optional code blocks, Hash inputs, and provides
+    # standardized error handling for malformed data.
+    # @deprecated Scoring is now handled internally by {SkillBench::Judge} and {SkillBench::DeltaReport}.
+    class JudgeScoreParserService
+      PARSE_ERROR = 'Failed to parse judge score'
+
+      # Parses a judge score response into a standardized format.
+      #
+      # @param judge_score [String, Hash, nil] Raw judge score response. Can be:
+      #   - A JSON string (with or without markdown code blocks)
+      #   - A Hash (with string or symbol keys)
+      #   - nil (which will result in an error response)
+      # @return [Hash] Standardized response hash with format:
+      #   - { success: true, response: Hash } on success
+      #   - { success: false, response: { error: { message: String } } on failure
+      # @raise [JSON::ParserError] raised when the judge_score string contains invalid JSON (rescued internally)
+      def self.call(judge_score)
+        new(judge_score).call
+      end
+
+      # @param judge_score [String, Hash, nil] Raw judge score response
+      def initialize(judge_score)
+        @judge_score = judge_score
+      end
+
+      # @return [Hash] { success: Boolean, response: Hash }
+      # @raise [JSON::ParserError] raised when the judge_score string contains invalid JSON (rescued internally)
+      def call
+        case @judge_score
+        when String
+          parsed = parse_string_input
+          parsed ? { success: true, response: parsed } : error_response
+        when Hash
+          { success: true, response: @judge_score.transform_keys(&:to_s) }
+        else
+          error_response
+        end
+      end
+
+      private
+
+      def error_response
+        { success: false, response: { error: { message: PARSE_ERROR } } }
+      end
+
+      # @return [Hash, nil] Parsed JSON hash or nil if parsing fails or not a Hash
+      def parse_string_input
+        # Remove markdown code blocks and extra whitespace
+        cleaned_score = @judge_score.strip
+        cleaned_score = cleaned_score.gsub(/\A```json\s*|\s*```\z/, '').strip
+
+        parsed = JSON.parse(cleaned_score)
+        parsed.is_a?(Hash) ? parsed : nil
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/skill_bench/services/junit_formatter.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module SkillBench
+  module Services
+    # Formats evaluation results as JUnit XML.
+    class JUnitFormatter
+      # Format result as JUnit XML.
+      #
+      # Supports both legacy format (result[:pass]) and modern DeltaReport format.
+      #
+      # @param result [Hash] Eval result.
+      # @return [String] JUnit XML-formatted string.
+      def self.format(result)
+        report = result.dig(:response, :report)
+        verdict = report.respond_to?(:verdict) ? report.verdict : result[:pass]
+        eval_name = CGI.escapeHTML(result[:eval_name].to_s)
+
+        if verdict
+          <<~XML
+            <?xml version="1.0"?>
+            <testsuite name="SkillBench" tests="1" failures="0">
+              <testcase name="#{eval_name}" classname="SkillBench"/>
+            </testsuite>
+          XML
+        else
+          score = report.respond_to?(:context_total) ? report.context_total : result[:score]
+          escaped_score = CGI.escapeHTML(score.to_s)
+          <<~XML
+            <?xml version="1.0"?>
+            <testsuite name="SkillBench" tests="1" failures="1">
+              <testcase name="#{eval_name}" classname="SkillBench">
+                <failure message="Score: #{escaped_score}">Eval failed</failure>
+              </testcase>
+            </testsuite>
+          XML
+        end
+      end
+    end
+  end
+end