qualspec 0.0.1

data/lib/qualspec/suite/reporter.rb

@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Qualspec
+  module Suite
+    class Reporter
+      def initialize(results, show_responses: false)
+        @results = results
+        @show_responses = show_responses
+      end
+
+      def to_stdout
+        output = []
+        output << header
+        output << ''
+        output << summary_table
+        output << ''
+        output << timing_section if timing?
+        output << ''
+        output << scenario_breakdown
+        output << ''
+        output << responses_section if @show_responses
+        output << winner_announcement
+
+        output.compact.join("\n")
+      end
+
+      def to_json(pretty: true)
+        if pretty
+          JSON.pretty_generate(@results.to_h)
+        else
+          JSON.generate(@results.to_h)
+        end
+      end
+
+      def write_json(path)
+        File.write(path, to_json)
+      end
+
+      private
+
+      def header
+        lines = []
+        lines << '=' * 60
+        lines << @results.suite_name.center(60)
+        lines << '=' * 60
+        lines.join("\n")
+      end
+
+      def summary_table
+        scores = @results.scores_by_candidate
+        return 'No results' if scores.empty?
+
+        candidates = scores.keys
+        max_name = [candidates.map(&:length).max, 10].max
+
+        # Count scenario wins
+        wins = count_wins
+
+        lines = []
+        lines << '## Summary'
+        lines << ''
+
+        header = "| #{'Candidate'.ljust(max_name)} | Score |  Wins | Pass Rate |"
+        lines << header
+        lines << "|#{'-' * (max_name + 2)}|-------|-------|-----------|"
+
+        sorted = scores.sort_by { |_, v| -v[:avg_score] }
+
+        sorted.each do |candidate, stats|
+          score = stats[:avg_score].to_s.rjust(5)
+          win_count = (wins[candidate] || 0).to_s.rjust(5)
+          pass_rate = "#{stats[:pass_rate]}%".rjust(8)
+
+          lines << "| #{candidate.ljust(max_name)} | #{score} | #{win_count} | #{pass_rate} |"
+        end
+
+        lines.join("\n")
+      end
+
+      def count_wins
+        wins = Hash.new(0)
+        @results.evaluations.each do |eval|
+          wins[eval[:candidate]] += 1 if eval[:winner] == true
+        end
+        wins
+      end
+
+      def timing?
+        !@results.timing.empty?
+      end
+
+      def timing_section
+        timing = @results.timing_by_candidate
+        return nil if timing.empty?
+
+        costs = @results.costs
+
+        lines = []
+        lines << '## Performance'
+        lines << ''
+
+        sorted = timing.sort_by { |_, v| v[:avg_ms] }
+
+        sorted.each do |candidate, stats|
+          line = "  #{candidate}: #{format_duration(stats[:avg_ms])} avg"
+          line += " (#{format_duration(stats[:total_ms])} total)"
+
+          line += " - $#{format_cost(costs[candidate])}" if costs[candidate]&.positive?
+
+          lines << line
+        end
+
+        lines.join("\n")
+      end
+
+      def format_duration(milliseconds)
+        if milliseconds >= 1000
+          "#{(milliseconds / 1000.0).round(2)}s"
+        else
+          "#{milliseconds}ms"
+        end
+      end
+
+      def format_cost(cost)
+        if cost < 0.01
+          format('%.4f', cost)
+        else
+          format('%.2f', cost)
+        end
+      end
+
+      def scenario_breakdown
+        by_scenario = @results.scores_by_scenario
+        return '' if by_scenario.empty?
+
+        candidates = @results.scores_by_candidate.keys
+
+        lines = []
+        lines << '## By Scenario'
+        lines << ''
+
+        by_scenario.each do |scenario, candidate_scores|
+          # Find winner for this scenario
+          winner = find_scenario_winner(scenario)
+          winner_label = if winner == :tie
+                           ' [TIE]'
+                         else
+                           winner ? " [Winner: #{winner}]" : ''
+                         end
+
+          lines << "### #{scenario}#{winner_label}"
+
+          candidates.each do |candidate|
+            stats = candidate_scores[candidate]
+            next unless stats
+
+            score_bar = score_visualization(stats[:score])
+            timing_info = format_scenario_timing(candidate, scenario)
+            win_marker = winner == candidate ? ' *' : ''
+
+            line = "  #{candidate}: #{score_bar} #{stats[:score]}/10#{win_marker}"
+            line += " #{timing_info}" if timing_info
+
+            lines << line
+          end
+
+          lines << ''
+        end
+
+        lines.join("\n")
+      end
+
+      def find_scenario_winner(scenario)
+        scenario_evals = @results.evaluations.select { |e| e[:scenario] == scenario }
+        winner_eval = scenario_evals.find { |e| e[:winner] == true }
+        return winner_eval[:candidate] if winner_eval
+
+        tie_eval = scenario_evals.find { |e| e[:winner] == :tie }
+        return :tie if tie_eval
+
+        nil
+      end
+
+      def format_scenario_timing(candidate, scenario)
+        duration = @results.timing.dig(candidate, scenario)
+        return nil unless duration
+
+        "[#{format_duration(duration)}]"
+      end
+
+      def score_visualization(score)
+        filled = score.to_f.round
+        empty = 10 - filled
+        "[#{'█' * filled}#{'░' * empty}]"
+      end
+
+      def responses_section
+        responses = @results.responses
+        return nil if responses.empty?
+
+        lines = []
+        lines << '## Responses'
+        lines << ''
+
+        # Group by scenario
+        scenarios = responses.values.first&.keys || []
+
+        scenarios.each do |scenario|
+          lines << "### #{scenario}"
+          lines << ''
+
+          responses.each do |candidate, candidate_responses|
+            response = candidate_responses[scenario]
+            next unless response
+
+            lines << "**#{candidate}:**"
+            lines << '```'
+            lines << response.to_s.strip[0..500]
+            lines << '...' if response.to_s.length > 500
+            lines << '```'
+            lines << ''
+          end
+        end
+
+        lines.join("\n")
+      end
+
+      def winner_announcement
+        scores = @results.scores_by_candidate
+        return '' if scores.empty?
+
+        wins = count_wins
+        sorted = scores.sort_by { |_, v| -v[:avg_score] }
+        winner = sorted.first
+        runner_up = sorted[1]
+
+        lines = []
+        lines << '-' * 60
+
+        if sorted.size == 1
+          lines << "Result: #{winner[0]} scored #{winner[1][:avg_score]}/10"
+        elsif winner[1][:avg_score] == runner_up&.dig(1, :avg_score)
+          tied = sorted.take_while { |_, v| v[:avg_score] == winner[1][:avg_score] }
+          lines << "Result: TIE between #{tied.map(&:first).join(', ')}"
+          lines << "        All scored #{winner[1][:avg_score]}/10 average"
+        else
+          margin = (winner[1][:avg_score] - runner_up[1][:avg_score]).round(2)
+          win_count = wins[winner[0]] || 0
+          lines << "Winner: #{winner[0]}"
+          avg_score = winner[1][:avg_score]
+          pass_rate = winner[1][:pass_rate]
+          lines << "        #{avg_score}/10 avg | #{win_count} scenario wins | #{pass_rate}% pass rate"
+          lines << "        Beat #{runner_up[0]} by #{margin} points"
+        end
+
+        timing = @results.timing_by_candidate
+        if timing.size > 1
+          fastest = timing.min_by { |_, v| v[:avg_ms] }
+          slowest = timing.max_by { |_, v| v[:avg_ms] }
+          if fastest[0] != slowest[0]
+            speedup = (slowest[1][:avg_ms].to_f / fastest[1][:avg_ms]).round(1)
+            lines << ''
+            lines << "Fastest: #{fastest[0]} (#{format_duration(fastest[1][:avg_ms])} avg)"
+            lines << "         #{speedup}x faster than #{slowest[0]}"
+          end
+        end
+
+        lines.join("\n")
+      end
+    end
+  end
+end

data/lib/qualspec/suite/runner.rb

@@ -0,0 +1,261 @@
+# frozen_string_literal: true
+
+require 'time'
+
+module Qualspec
+  module Suite
+    class Runner
+      attr_reader :definition, :results
+
+      def initialize(definition)
+        @definition = definition.is_a?(String) ? Suite.find(definition) : definition
+        @results = Results.new(@definition.name)
+        @judge = Qualspec.judge
+      end
+
+      def run(progress: true)
+        total_scenarios = @definition.scenarios_list.size
+        current = 0
+
+        # Process by scenario - collect all candidate responses, then judge together
+        @definition.scenarios_list.each do |scenario|
+          current += 1
+          log_scenario_progress(current, total_scenarios, scenario) if progress
+
+          run_scenario_comparison(scenario, progress: progress)
+
+          yield(@results) if block_given?
+        end
+
+        @results
+      end
+
+      private
+
+      def run_scenario_comparison(scenario, progress: false)
+        responses = {}
+        errors = {}
+
+        # Phase 1: Collect all candidate responses
+        @definition.candidates_list.each do |candidate|
+          log_candidate_progress(candidate, scenario, 'generating') if progress
+
+          response_data = generate_response(candidate, scenario)
+
+          if response_data[:error]
+            log_error(candidate, scenario, response_data[:error])
+            errors[candidate.name] = response_data[:error]
+          else
+            response = response_data[:response]
+            response_content = response.is_a?(Client::Response) ? response.content : response
+
+            responses[candidate.name] = response_content
+
+            @results.record_response(
+              candidate: candidate.name,
+              scenario: scenario.name,
+              response: response_content,
+              duration_ms: response.is_a?(Client::Response) ? response.duration_ms : response_data[:duration_ms],
+              cost: response.is_a?(Client::Response) ? response.cost : nil
+            )
+          end
+        end
+
+        # Phase 2: Judge all responses together (if we have any)
+        if responses.any?
+          log_candidate_progress(nil, scenario, 'judging') if progress
+
+          context = build_context(scenario)
+          criteria = scenario.all_criteria
+
+          # Use comparison mode for multiple candidates, single eval for one
+          if responses.size == 1
+            candidate, response = responses.first
+            evaluation = @judge.evaluate(
+              response: response,
+              criterion: criteria.join("\n"),
+              context: context
+            )
+            @results.record_evaluation(
+              candidate: candidate,
+              scenario: scenario.name,
+              criteria: criteria,
+              evaluation: evaluation,
+              winner: true # Only candidate wins by default
+            )
+          else
+            evaluations = @judge.evaluate_comparison(
+              responses: responses,
+              criteria: criteria,
+              context: context
+            )
+
+            evaluations.each do |candidate, evaluation|
+              @results.record_evaluation(
+                candidate: candidate,
+                scenario: scenario.name,
+                criteria: criteria,
+                evaluation: evaluation,
+                winner: evaluation.scenario_winner
+              )
+            end
+          end
+        end
+
+        # Record errors for failed candidates
+        errors.each do |candidate, error_message|
+          @results.record_evaluation(
+            candidate: candidate,
+            scenario: scenario.name,
+            criteria: scenario.all_criteria,
+            evaluation: Evaluation.new(
+              criterion: scenario.all_criteria.join("\n"),
+              score: 0,
+              pass: false,
+              error: error_message
+            )
+          )
+        end
+      end
+
+      def generate_response(candidate, scenario)
+        start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
+        response = candidate.generate_response(
+          prompt: scenario.prompt_text,
+          system_prompt: scenario.system_prompt
+        )
+
+        duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
+
+        { response: response, duration_ms: duration_ms }
+      rescue StandardError => e
+        { error: e.message }
+      end
+
+      def build_context(scenario)
+        parts = []
+        parts << "System prompt: #{scenario.system_prompt}" if scenario.system_prompt
+        parts << "User prompt: #{scenario.prompt_text}"
+        parts << scenario.context if scenario.context
+        parts.join("\n\n")
+      end
+
+      def log_scenario_progress(current, total, scenario)
+        pct = ((current.to_f / total) * 100).round
+        $stderr.print "\r[#{pct}%] Scenario: #{scenario.name}".ljust(60)
+      end
+
+      def log_candidate_progress(candidate, _scenario, phase)
+        name = candidate&.name || 'all'
+        $stderr.print "\r       #{name}: #{phase}...".ljust(60)
+      end
+
+      def log_error(candidate, scenario, error)
+        warn "\n  ERROR (#{candidate.name}/#{scenario.name}): #{error[0..100]}"
+      end
+    end
+
+    # Results container
+    class Results
+      attr_reader :suite_name, :evaluations, :responses, :started_at, :finished_at, :timing, :costs
+
+      def initialize(suite_name)
+        @suite_name = suite_name
+        @evaluations = []
+        @responses = {}
+        @timing = {}
+        @costs = {}
+        @started_at = Time.now
+        @finished_at = nil
+      end
+
+      def record_response(candidate:, scenario:, response:, duration_ms: nil, cost: nil)
+        @responses[candidate] ||= {}
+        @responses[candidate][scenario] = response
+
+        if duration_ms
+          @timing[candidate] ||= {}
+          @timing[candidate][scenario] = duration_ms
+        end
+
+        return unless cost&.positive?
+
+        @costs[candidate] ||= 0.0
+        @costs[candidate] += cost
+      end
+
+      def record_evaluation(candidate:, scenario:, criteria:, evaluation:, winner: nil)
+        @evaluations << {
+          candidate: candidate,
+          scenario: scenario,
+          criteria: criteria,
+          criteria_count: Array(criteria).size,
+          score: evaluation.score,
+          pass: evaluation.pass?,
+          reasoning: evaluation.reasoning,
+          error: evaluation.error,
+          winner: winner
+        }
+      end
+
+      def finish!
+        @finished_at = Time.now
+      end
+
+      def scores_by_candidate
+        @evaluations.group_by { |e| e[:candidate] }.transform_values do |evals|
+          passed = evals.count { |e| e[:pass] }
+          total = evals.size
+          avg_score = total.positive? ? evals.sum { |e| e[:score] }.to_f / total : 0
+
+          {
+            passed: passed,
+            total: total,
+            pass_rate: total.positive? ? (passed.to_f / total * 100).round(1) : 0,
+            avg_score: avg_score.round(2)
+          }
+        end
+      end
+
+      def timing_by_candidate
+        @timing.transform_values do |scenarios|
+          total_ms = scenarios.values.sum
+          avg_ms = !scenarios.empty? ? total_ms / scenarios.size : 0
+          {
+            total_ms: total_ms,
+            avg_ms: avg_ms.round,
+            count: scenarios.size
+          }
+        end
+      end
+
+      def scores_by_scenario
+        @evaluations.group_by { |e| e[:scenario] }.transform_values do |evals|
+          evals.group_by { |e| e[:candidate] }.transform_values do |candidate_evals|
+            eval_data = candidate_evals.first
+            {
+              score: eval_data[:score],
+              pass: eval_data[:pass],
+              reasoning: eval_data[:reasoning]
+            }
+          end
+        end
+      end
+
+      def to_h
+        {
+          suite_name: @suite_name,
+          started_at: @started_at.iso8601,
+          finished_at: @finished_at&.iso8601,
+          summary: scores_by_candidate,
+          timing: timing_by_candidate,
+          costs: @costs,
+          by_scenario: scores_by_scenario,
+          evaluations: @evaluations,
+          responses: @responses
+        }
+      end
+    end
+  end
+end

data/lib/qualspec/suite/scenario.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Qualspec
+  module Suite
+    class Scenario
+      attr_reader :name, :prompt_text, :system_prompt, :evaluations, :rubric_name, :context
+
+      def initialize(name, &block)
+        @name = name
+        @prompt_text = nil
+        @system_prompt = nil
+        @evaluations = []
+        @rubric_name = nil
+        @context = nil
+
+        instance_eval(&block) if block_given? # rubocop:disable Style/EvalWithLocation -- DSL pattern requires eval
+      end
+
+      # DSL methods
+      def prompt(text)
+        @prompt_text = text
+      end
+
+      def system(text)
+        @system_prompt = text
+      end
+
+      # DSL method to add evaluation criteria
+      def criterion(text)
+        @evaluations << text
+      end
+
+      # Alias for backwards compatibility
+      alias evaluate criterion
+
+      def rubric(name)
+        @rubric_name = name
+      end
+
+      def with_context(text)
+        @context = text
+      end
+
+      # Get all criteria to evaluate (from explicit evals + rubric)
+      def all_criteria
+        criteria = @evaluations.dup
+
+        if @rubric_name
+          rubric_obj = Rubric.find(@rubric_name)
+          criteria.concat(rubric_obj.criteria)
+        end
+
+        criteria
+      end
+    end
+  end
+end

data/lib/qualspec/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Qualspec
+  VERSION = '0.0.1'
+end

data/lib/qualspec.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require_relative 'qualspec/version'
+
+module Qualspec
+  class Error < StandardError; end
+end
+
+require_relative 'qualspec/configuration'
+require_relative 'qualspec/client'
+require_relative 'qualspec/evaluation'
+require_relative 'qualspec/rubric'
+require_relative 'qualspec/judge'
+require_relative 'qualspec/builtin_rubrics'
+require_relative 'qualspec/suite/candidate'
+require_relative 'qualspec/suite/scenario'
+require_relative 'qualspec/suite/behavior'
+require_relative 'qualspec/suite/dsl'
+require_relative 'qualspec/suite/runner'
+require_relative 'qualspec/suite/reporter'
+require_relative 'qualspec/suite/html_reporter'
+require_relative 'qualspec/suite/builtin_behaviors'
+require_relative 'qualspec/recorder'
+
+module Qualspec
+  class << self
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def configure
+      yield(configuration)
+    end
+
+    def reset!
+      @configuration = nil
+      @client = nil
+      @judge = nil
+      Rubric.clear!
+      Suite.clear!
+      Suite::Behavior.clear!
+    end
+
+    def client
+      @client ||= Client.new(configuration)
+    end
+
+    def judge
+      @judge ||= Judge.new
+    end
+
+    # Convenience method for defining rubrics
+    def define_rubric(name, &block)
+      Rubric.define(name, &block)
+    end
+
+    # Convenience method for defining behaviors
+    def define_behavior(name, &block)
+      Suite::Behavior.define(name, &block)
+    end
+
+    # Convenience method for defining evaluation suites
+    def evaluation(name, &block)
+      Suite.define(name, &block)
+    end
+
+    # Run an evaluation suite
+    def run(suite_name, progress: true, output: :stdout, json_path: nil, html_path: nil, show_responses: false,
+            load_builtins: true)
+      # Load builtins (idempotent, can be called multiple times)
+      if load_builtins
+        BuiltinRubrics.load!
+        Suite::BuiltinBehaviors.load!
+      end
+
+      suite = Suite.find(suite_name)
+      runner = Suite::Runner.new(suite)
+
+      results = runner.run(progress: progress)
+      results.finish!
+
+      reporter = Suite::Reporter.new(results, show_responses: show_responses)
+
+      case output
+      when :stdout
+        puts reporter.to_stdout
+      when :json
+        puts reporter.to_json
+      when :silent
+        # nothing
+      end
+
+      reporter.write_json(json_path) if json_path
+
+      if html_path
+        html_reporter = Suite::HtmlReporter.new(results)
+        html_reporter.write(html_path)
+      end
+
+      results
+    end
+  end
+end

data/sig/qualspec.rbs

@@ -0,0 +1,4 @@
+module Qualspec
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end