ruby-skill-bench 0.1.0

data/lib/skill_bench/history_recorder/history_file.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'fileutils'
+
+module SkillBench
+  class HistoryRecorder
+    # Handles atomic read/write of benchmark history JSON files.
+    class HistoryFile
+      # Loads history from the given path.
+      #
+      # @param path [String] path to the JSON history file
+      # @return [Array<Hash>] parsed history entries
+      def self.load(path)
+        new.load(path)
+      end
+
+      # Writes history data atomically to the given path.
+      #
+      # @param path [String] target file path
+      # @param data [Array<Hash>] history entries to serialize
+      # @return [void]
+      def self.write(path, data)
+        new.write(path, data)
+      end
+
+      # Loads history from the given path.
+      #
+      # @param path [String] path to the JSON history file
+      # @return [Array<Hash>] parsed history entries
+      def load(path)
+        return [] unless File.exist?(path)
+
+        JSON.parse(File.read(path), symbolize_names: true)
+      rescue JSON::ParserError => e
+        SkillBench::ErrorLogger.log_error(e, 'corrupted benchmarks.json')
+        []
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'HistoryRecorder')
+        []
+      end
+
+      # Writes history data atomically using a temp file and rename.
+      #
+      # @param path [String] target file path
+      # @param data [Array<Hash>] history entries to serialize
+      # @return [void]
+      def write(path, data)
+        dir = File.dirname(path)
+        FileUtils.mkpath(dir)
+
+        temp_path = "#{path}.tmp.#{Process.pid}"
+        File.open(temp_path, File::WRONLY | File::CREAT | File::TRUNC, 0o644) do |file|
+          file.flock(File::LOCK_EX)
+          file.write(JSON.pretty_generate(data))
+          file.fsync
+        end
+        File.rename(temp_path, path)
+        logger&.info("History recorded to #{path}")
+      end
+
+      private
+
+      def logger
+        ::Rails.logger
+      rescue NameError
+        nil
+      end
+    end
+  end
+end

data/lib/skill_bench/history_recorder/history_path_resolver.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+
+module SkillBench
+  class HistoryRecorder
+    # Resolves the best writable path for storing benchmark history.
+    # Checks env var, cwd, local share, and XDG data home in order.
+    class HistoryPathResolver
+      # Finds the best writable path for the history file.
+      #
+      # @return [String, nil] writable path or nil if none found
+      def self.resolve
+        new.resolve
+      end
+
+      # Finds the best writable path for the history file.
+      #
+      # @return [String, nil] writable path or nil if none found
+      def resolve
+        env_path || cwd_path || local_path || xdg_path || begin
+          warn('Warning: Could not find writable location for benchmarks.json')
+          nil
+        end
+      end
+
+      private
+
+      def env_path
+        raw = ENV.fetch('SKILL_BENCH_HISTORY_FILE', '').to_s.strip
+        return nil if raw.empty?
+
+        expanded = File.expand_path(raw)
+        unless contained?(expanded)
+          warn "Warning: SKILL_BENCH_HISTORY_FILE '#{raw}' rejected (outside allowed directories or not writable)."
+          return nil
+        end
+        return nil unless prepare_and_writable?(expanded)
+
+        expanded
+      end
+
+      def cwd_path
+        path = File.join(Dir.pwd, 'benchmarks.json')
+        return nil unless File.writable?(File.dirname(path))
+
+        path
+      end
+
+      def local_path
+        path = File.join(Dir.home, '.local', 'share', 'skill_bench', 'benchmarks.json')
+        return nil unless prepare_and_writable?(path)
+
+        path
+      end
+
+      def xdg_path
+        xdg_data_home = ENV.fetch('XDG_DATA_HOME', File.join(Dir.home, '.local', 'share'))
+        path = File.join(xdg_data_home, 'skill_bench', 'benchmarks.json')
+        return nil unless prepare_and_writable?(path)
+
+        path
+      end
+
+      def contained?(path)
+        path_with_sep = path + File::SEPARATOR
+        allowed_prefixes.any? do |prefix|
+          expanded_prefix = File.expand_path(prefix) + File::SEPARATOR
+          path_with_sep.start_with?(expanded_prefix) || path == expanded_prefix.chomp(File::SEPARATOR)
+        end
+      end
+
+      def allowed_prefixes
+        [Dir.pwd, File.join(Dir.home, '.local', 'share', 'skill_bench')]
+      end
+
+      def prepare_and_writable?(path)
+        dir_name = File.dirname(path)
+        FileUtils.mkpath(dir_name)
+        File.writable?(dir_name)
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'HistoryRecorder')
+        false
+      end
+    end
+  end
+end

data/lib/skill_bench/history_recorder/persistence_service.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module SkillBench
+  class HistoryRecorder
+    # Orchestrates recording evaluation results to the history file.
+    # Thin service that delegates path resolution and file I/O to
+    # HistoryPathResolver and HistoryFile respectively.
+    class PersistenceService
+      # Records evaluation results into a historical benchmarks file.
+      #
+      # @param results [Hash] The results from a Runner.call.
+      # @param source_path [String] The resolved source path used for the evaluation.
+      # @param model [String] The model name used for the evaluation.
+      # @return [Boolean] true if recorded successfully, false otherwise.
+      def self.record(results, source_path:, model:)
+        return false unless results[:success]
+
+        history_file = HistoryPathResolver.resolve
+        return false unless history_file
+
+        history = HistoryFile.load(history_file)
+        entry = {
+          timestamp: Time.now.iso8601,
+          source_path: source_path,
+          model: model,
+          summary: SummaryService.summarize(results[:tasks])
+        }
+
+        history << entry
+        HistoryFile.write(history_file, history)
+        true
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'HistoryRecorder')
+        false
+      end
+    end
+  end
+end

data/lib/skill_bench/history_recorder/summary_service.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module SkillBench
+  class HistoryRecorder
+    # Service object for summarizing evaluation results.
+    # Handles score normalization and statistical calculations.
+    # Follows Single Responsibility Principle by isolating summary concerns.
+    class SummaryService
+      # Summarizes the results of multiple tasks.
+      #
+      # @param tasks [Array<Hash>] The list of task results.
+      # @return [Hash] A summary of scores including averages and improvement.
+      def self.summarize(tasks)
+        return {} if Array(tasks).empty?
+
+        scores = tasks.map { |task| normalize_score(task[:judge_score]) }
+        calculate_summary(scores)
+      end
+
+      # Normalizes the raw judge score into a standardized Hash.
+      #
+      # @param raw_score [String, Hash, nil] The raw score from the judge.
+      # @return [Hash] The normalized score with :baseline_score and :context_score.
+      # @raise [JSON::ParserError] raised when the judge_score string contains invalid JSON (rescued internally).
+      def self.normalize_score(raw_score)
+        return {} unless raw_score
+        return raw_score if raw_score.is_a?(Hash)
+
+        begin
+          JSON.parse(raw_score, symbolize_names: true)
+        rescue JSON::ParserError
+          {}
+        end
+      end
+
+      # Calculates statistical summary from a list of normalized scores.
+      #
+      # @param scores [Array<Hash>] List of normalized scores.
+      # @return [Hash] Summary statistics.
+      def self.calculate_summary(scores)
+        count = scores.size
+        baseline_total = 0.0
+        context_total = 0.0
+
+        scores.each do |score|
+          baseline_total += (score[:baseline_score] || 0).to_f
+          context_total += (score[:context_score] || 0).to_f
+        end
+
+        {
+          task_count: count,
+          average_baseline: (baseline_total / count).round(2),
+          average_context: (context_total / count).round(2),
+          improvement: ((context_total - baseline_total) / count).round(2)
+        }
+      end
+    end
+  end
+end

data/lib/skill_bench/history_recorder.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative 'history_recorder/persistence_service'
+require_relative 'history_recorder/summary_service'
+require_relative 'history_recorder/history_path_resolver'
+require_relative 'history_recorder/history_file'
+
+# Top-level namespace for the Rails Agent Evaluator.
+module SkillBench
+  # Records evaluation results into a historical benchmarks file.
+  # Delegates to specialized services following Single Responsibility Principle.
+  class HistoryRecorder
+    # The default file where historical benchmarks are stored.
+    HISTORY_FILE = 'benchmarks.json'
+
+    # Records evaluation results into a historical benchmarks file.
+    # Delegates to PersistenceService.
+    def self.record(results, source_path:, model:)
+      PersistenceService.record(results, source_path: source_path, model: model)
+    end
+
+    # Loads existing history from the benchmarks file.
+    # Delegates to HistoryFile.
+    def self.load_history(path = HISTORY_FILE)
+      HistoryFile.load(path)
+    end
+
+    # Summarizes the results of multiple tasks.
+    # Delegates to SummaryService.
+    def self.summarize(tasks)
+      SummaryService.summarize(tasks)
+    end
+
+    # Logs errors with backtrace.
+    # Delegates to ErrorLogger.
+    def self.log_error(exception)
+      SkillBench::ErrorLogger.log_error(exception, 'HistoryRecorder')
+    end
+  end
+end

data/lib/skill_bench/interactive.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'skill_bench/commands/run'
+
+# Provides interactive CLI mode using gum-like menu system
+module SkillBench
+  # Interactive CLI module for skill-bench
+  module Interactive
+    # Run the interactive CLI mode
+    # @return [Hash, nil] Result from Run.run, or nil if user exits
+    def self.run
+      choice = gum_choose
+      return nil unless choice
+
+      case choice
+      when 'Run Eval'
+        eval_name = select_eval
+        skill_name = select_skill
+        provider_name = select_provider
+
+        return nil unless eval_name && skill_name && provider_name
+
+        SkillBench::Commands::Run.run(
+          eval_name: eval_name,
+          skill_name: skill_name,
+          provider_name: provider_name
+        )
+      when 'Exit'
+        exit 0
+      end
+    end
+
+    # Display main menu using gum
+    # @return [String, nil] User's choice or nil
+    # @raise [NotImplementedError] Raised when gum integration is not enabled
+    def self.gum_choose
+      raise NotImplementedError, 'Interactive selection not implemented; enable gum integration'
+    end
+
+    # Select an eval from available evals
+    # @return [String, nil] Eval name or nil
+    # @raise [NotImplementedError] Raised when gum integration is not enabled
+    def self.select_eval
+      raise NotImplementedError, 'Interactive selection not implemented; enable gum integration'
+    end
+
+    # Select a skill from available skills
+    # @return [String, nil] Skill name or nil
+    # @raise [NotImplementedError] Raised when gum integration is not enabled
+    def self.select_skill
+      raise NotImplementedError, 'Interactive selection not implemented; enable gum integration'
+    end
+
+    # Select a provider from available providers
+    # @return [String, nil] Provider name or nil
+    # @raise [NotImplementedError] Raised when gum integration is not enabled
+    def self.select_provider
+      raise NotImplementedError, 'Interactive selection not implemented; enable gum integration'
+    end
+  end
+end

data/lib/skill_bench/judge/judge.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require_relative '../client'
+
+module SkillBench
+  module Judge
+    # Responsible for evaluating AI-generated code modifications.
+    #
+    # Accepts a structured judge prompt, calls the LLM client,
+    # and parses the response into a Judge::Response with per-dimension scores.
+    class Judge
+      # System prompt sent to the LLM judge defining its role and output format.
+      SYSTEM_PROMPT = 'You are an objective judge evaluating AI coding models. ' \
+                      'Your goal is to score responses based strictly on the provided criteria. ' \
+                      'Return only valid JSON.'
+
+      # Evaluates agent output via the LLM judge.
+      #
+      # @param prompt [String] The structured judge prompt.
+      # @param client_params [Hash] Optional parameters to pass to the client.
+      # @return [Hash] with :success [Boolean] and :response containing Judge::Response or error.
+      def self.call(prompt:, client_params: {})
+        new(prompt:, client_params:).call
+      end
+
+      # @param prompt [String] The structured judge prompt.
+      # @param client_params [Hash] Optional client parameters.
+      def initialize(prompt:, client_params:)
+        @prompt = prompt
+        @client_params = client_params
+      end
+
+      # Executes the evaluation process via the LLM client.
+      #
+      # @return [Hash] Service response with Judge::Response or error.
+      def call
+        judge_result = Client.call(
+          system_prompt: SYSTEM_PROMPT,
+          messages: [{ role: 'user', content: prompt }],
+          **client_params
+        )
+
+        return judge_result unless judge_result[:success]
+
+        content = extract_content(judge_result)
+        return empty_response_result unless content
+
+        Response.call(json: content)
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'Judge Evaluation Error')
+        { success: false, response: { error: { message: e.message } } }
+      end
+
+      private
+
+      attr_reader :prompt, :client_params
+
+      def extract_content(judge_result)
+        response = judge_result[:response]
+        message = response[:message] || response['message']
+        return nil unless message.is_a?(Hash)
+
+        message[:content] || message['content']
+      end
+
+      def empty_response_result
+        { success: false, response: { error: { message: 'Empty response from judge' } } }
+      end
+    end
+  end
+end

data/lib/skill_bench/judge/prompt.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module SkillBench
+  module Judge
+    # Builds structured prompts for the LLM judge.
+    #
+    # Assembles task description, evaluation criteria, skill context,
+    # and agent output into a single prompt for blind scoring.
+    class Prompt
+      # Builds the judge prompt.
+      #
+      # @param task [String] The task description from task.md.
+      # @param criteria [SkillBench::Criteria] The eval criteria with dimensions.
+      # @param skill_context [String, nil] XML-wrapped skill context (nil for baseline runs).
+      # @param agent_output [String] Git diff and agent summary.
+      # @return [Hash] Service response with prompt or error.
+      def self.call(task:, criteria:, skill_context:, agent_output:)
+        new(task:, criteria:, skill_context:, agent_output:).call
+      end
+
+      # @param task [String] The task description.
+      # @param criteria [SkillBench::Criteria] The eval criteria.
+      # @param skill_context [String, nil] The skill context XML (nil for baseline runs).
+      # @param agent_output [String] The agent output.
+      def initialize(task:, criteria:, skill_context:, agent_output:)
+        @task = task
+        @criteria = criteria
+        @skill_context = skill_context
+        @agent_output = agent_output
+      end
+
+      # Assembles and returns the judge prompt.
+      #
+      # @return [Hash] Service response with prompt or error.
+      def call
+        return missing_task_result if task.nil? || task.strip.empty?
+        return missing_criteria_result if criteria.nil?
+        return missing_agent_output_result if agent_output.nil? || agent_output.to_s.strip.empty?
+        return missing_skill_context_result unless valid_skill_context?
+
+        prompt = assemble_prompt
+        { success: true, response: { prompt: prompt } }
+      rescue StandardError => e
+        SkillBench::ErrorLogger.log_error(e, 'Judge::Prompt Build Error')
+        { success: false, response: { error: { message: e.message } } }
+      end
+
+      private
+
+      attr_reader :task, :criteria, :skill_context, :agent_output
+
+      def missing_task_result
+        { success: false, response: { error: { message: 'Task is required' } } }
+      end
+
+      def missing_criteria_result
+        { success: false, response: { error: { message: 'Criteria is required' } } }
+      end
+
+      def missing_agent_output_result
+        { success: false, response: { error: { message: 'Agent output is required' } } }
+      end
+
+      def missing_skill_context_result
+        { success: false, response: { error: { message: 'Skill context is required' } } }
+      end
+
+      def valid_skill_context?
+        return true if skill_context.nil?
+
+        skill_context.is_a?(String) && !skill_context.strip.empty?
+      end
+
+      def assemble_prompt
+        sections = [
+          task_section,
+          criteria_section,
+          skill_context_section,
+          agent_output_section,
+          instructions_section
+        ]
+
+        sections.join("\n\n")
+      end
+
+      def task_section
+        "## Task\n\n#{task}"
+      end
+
+      def criteria_section
+        lines = ['## Criteria']
+        lines << "\nContext: #{criteria.context}"
+        lines << "\nDimensions:"
+
+        criteria.dimensions.each do |dim|
+          lines << "- #{dim.name}: max_score=#{dim.max_score}, description=#{dim.description}"
+        end
+
+        lines.join("\n")
+      end
+
+      def skill_context_section
+        "## Skill Context\n\n#{skill_context}"
+      end
+
+      def agent_output_section
+        "## Agent Output\n\n#{agent_output}"
+      end
+
+      def instructions_section
+        <<~INSTRUCTIONS
+          ## Instructions
+
+          Score each dimension independently. Return JSON with:
+          - "dimensions": object mapping each dimension name to { "score": number, "max_score": number, "reasoning": string }
+          - "overall_reasoning": string summarizing the evaluation
+        INSTRUCTIONS
+      end
+    end
+  end
+end