qualspec 0.0.1

data/lib/qualspec/recorder.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Qualspec
+  class Recorder
+    class << self
+      def available?
+        require 'vcr'
+        true
+      rescue LoadError
+        false
+      end
+
+      def setup(cassette_dir: '.qualspec_cassettes')
+        require_vcr!
+
+        VCR.configure do |config|
+          config.cassette_library_dir = cassette_dir
+          config.hook_into :faraday
+          config.default_cassette_options = {
+            record: :new_episodes,
+            match_requests_on: %i[method uri body]
+          }
+          # Filter out API keys
+          config.filter_sensitive_data('<API_KEY>') { Qualspec.configuration.api_key }
+        end
+        @configured = true
+      end
+
+      def configured?
+        @configured == true
+      end
+
+      def record(name, &block)
+        setup unless configured?
+        VCR.use_cassette(name, &block)
+      end
+
+      def playback(name, &block)
+        setup unless configured?
+        VCR.use_cassette(name, record: :none, &block)
+      end
+
+      private
+
+      def require_vcr!
+        require 'vcr'
+      rescue LoadError
+        raise Qualspec::Error, <<~MSG.strip
+          VCR gem is required for recording/playback features.
+          Add to your Gemfile: gem 'vcr'
+        MSG
+      end
+    end
+  end
+end

data/lib/qualspec/rspec/configuration.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Qualspec
+  module RSpec
+    # RSpec-specific configuration
+    #
+    # @example Configure in spec_helper.rb
+    #   Qualspec::RSpec.configure do |config|
+    #     config.default_threshold = 7
+    #     config.vcr_cassette_dir = "spec/cassettes/qualspec"
+    #     config.record_mode = :new_episodes
+    #   end
+    #
+    class Configuration
+      # Default pass threshold for evaluations (0-10)
+      attr_accessor :default_threshold
+
+      # Directory for VCR cassettes
+      attr_accessor :vcr_cassette_dir
+
+      # VCR recording mode (:new_episodes, :none, :all, :once)
+      attr_accessor :record_mode
+
+      # Whether to load builtin rubrics automatically
+      attr_accessor :load_builtins
+
+      def initialize
+        @default_threshold = 7
+        @vcr_cassette_dir = 'spec/cassettes/qualspec'
+        @record_mode = :new_episodes
+        @load_builtins = true
+      end
+    end
+
+    class << self
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      def configure
+        yield(configuration)
+      end
+
+      def reset_configuration!
+        @configuration = nil
+      end
+    end
+  end
+end

data/lib/qualspec/rspec/evaluation_result.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module Qualspec
+  module RSpec
+    # Wrapper around Qualspec::Evaluation with RSpec-friendly interface
+    # Provides clean methods and good failure message formatting
+    class EvaluationResult
+      attr_reader :evaluation, :criterion, :response, :threshold
+
+      def initialize(evaluation, criterion:, response:, threshold: nil)
+        @evaluation = evaluation
+        @criterion = criterion
+        @response = response
+        @threshold = threshold || Qualspec::RSpec.configuration.default_threshold
+      end
+
+      # Core result methods
+      def passing?
+        evaluation.pass?
+      end
+
+      def failing?
+        !passing?
+      end
+
+      def score
+        evaluation.score
+      end
+
+      def reasoning
+        evaluation.reasoning
+      end
+
+      def error?
+        !evaluation.error.nil?
+      end
+
+      def error
+        evaluation.error
+      end
+
+      # Comparison support
+      def winner?
+        evaluation.scenario_winner == true
+      end
+
+      def tie?
+        evaluation.scenario_winner == :tie
+      end
+
+      # Nice output for RSpec failure messages
+      def inspect
+        status = passing? ? 'PASS' : 'FAIL'
+        lines = [
+          "#<Qualspec::RSpec::EvaluationResult #{status}>",
+          "  Criterion: #{criterion}",
+          "  Score: #{score}/10 (threshold: #{threshold})",
+          "  Reasoning: #{reasoning}"
+        ]
+        lines << "  Error: #{error}" if error?
+        lines << "  Response (first 200 chars): #{response.to_s[0, 200]}..."
+        lines.join("\n")
+      end
+
+      # RSpec failure message formatting
+      def failure_message
+        <<~MSG
+          Expected response to pass qualspec evaluation, but it failed.
+
+          Criterion: #{criterion}
+          Score: #{score}/10 (needed #{threshold} to pass)
+          Reasoning: #{reasoning}
+          #{"Error: #{error}" if error?}
+          Response preview: #{response.to_s[0, 300]}#{'...' if response.to_s.length > 300}
+        MSG
+      end
+
+      def failure_message_when_negated
+        <<~MSG
+          Expected response to fail qualspec evaluation, but it passed.
+
+          Criterion: #{criterion}
+          Score: #{score}/10 (threshold: #{threshold})
+          Reasoning: #{reasoning}
+        MSG
+      end
+    end
+
+    # Result for comparative evaluations
+    class ComparisonResult
+      attr_reader :results, :winner, :criterion
+
+      def initialize(results, criterion:)
+        @results = results # Hash of name => EvaluationResult
+        @criterion = criterion
+        @winner = determine_winner
+      end
+
+      def [](name)
+        results[name.to_sym]
+      end
+
+      def tie?
+        winner == :tie
+      end
+
+      def winner?(name)
+        winner.to_s == name.to_s
+      end
+
+      def scores
+        results.transform_values(&:score)
+      end
+
+      def inspect
+        lines = ['#<Qualspec::RSpec::ComparisonResult>']
+        lines << "  Criterion: #{criterion}"
+        lines << "  Winner: #{winner}"
+        results.each do |name, result|
+          marker = winner?(name) ? '*' : ' '
+          lines << "  #{marker} #{name}: #{result.score}/10 - #{result.reasoning}"
+        end
+        lines.join("\n")
+      end
+
+      private
+
+      def determine_winner
+        # Check if any result has winner flag set
+        results.each do |name, result|
+          return name if result.winner?
+          return :tie if result.tie?
+        end
+
+        # Fallback: highest score wins
+        max_score = results.values.map(&:score).max
+        winners = results.select { |_, r| r.score == max_score }
+        winners.size == 1 ? winners.keys.first : :tie
+      end
+    end
+  end
+end

data/lib/qualspec/rspec/helpers.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require_relative 'evaluation_result'
+
+module Qualspec
+  module RSpec
+    # Helper methods for RSpec tests
+    # Include this in your spec_helper.rb:
+    #
+    #   RSpec.configure do |config|
+    #     config.include Qualspec::RSpec::Helpers
+    #   end
+    #
+    module Helpers
+      # Evaluate a response against a criterion or rubric
+      #
+      # @param response [String] The response to evaluate
+      # @param criterion [String, nil] The evaluation criterion (optional if using rubric)
+      # @param rubric [Symbol, nil] A pre-defined rubric name
+      # @param context [String, nil] Additional context for the judge
+      # @param threshold [Integer, nil] Pass threshold (default: 7)
+      # @return [EvaluationResult]
+      #
+      # @example With inline criterion
+      #   result = qualspec_evaluate(response, "responds in a friendly manner")
+      #   expect(result).to be_passing
+      #
+      # @example With rubric
+      #   result = qualspec_evaluate(response, rubric: :tool_calling)
+      #   expect(result.score).to be >= 8
+      #
+      # @example With context
+      #   result = qualspec_evaluate(response, "summarizes accurately",
+      #                              context: "User provided a news article")
+      #   expect(result).to be_passing
+      #
+      def qualspec_evaluate(response, criterion = nil, rubric: nil, context: nil, threshold: nil)
+        raise ArgumentError, 'Must provide either criterion or rubric:' if criterion.nil? && rubric.nil?
+
+        effective_threshold = threshold || Qualspec::RSpec.configuration.default_threshold
+        criterion_text, display_criterion = resolve_criterion(criterion, rubric)
+
+        evaluation = Qualspec.judge.evaluate(
+          response: response.to_s,
+          criterion: criterion_text,
+          context: context,
+          pass_threshold: effective_threshold
+        )
+
+        EvaluationResult.new(
+          evaluation,
+          criterion: display_criterion,
+          response: response,
+          threshold: effective_threshold
+        )
+      end
+
+      # Compare multiple responses against criteria
+      #
+      # @param responses [Hash] Hash of name => response
+      # @param criterion [String, Array<String>] Evaluation criteria
+      # @param context [String, nil] Additional context
+      # @param threshold [Integer, nil] Pass threshold (default: 7)
+      # @return [ComparisonResult]
+      #
+      # @example
+      #   responses = {
+      #     gpt4: gpt4_response,
+      #     claude: claude_response
+      #   }
+      #   result = qualspec_compare(responses, "explains clearly")
+      #   expect(result.winner).to eq(:claude)
+      #   expect(result[:gpt4].score).to be >= 7
+      #
+      def qualspec_compare(responses, criterion, context: nil, threshold: nil)
+        effective_threshold = threshold || Qualspec::RSpec.configuration.default_threshold
+        criteria_list = Array(criterion)
+        display_criterion = criteria_list.join('; ')
+
+        evaluations = Qualspec.judge.evaluate_comparison(
+          responses: responses.transform_values(&:to_s),
+          criteria: criteria_list,
+          context: context,
+          pass_threshold: effective_threshold
+        )
+
+        results = wrap_comparison_results(evaluations, responses, display_criterion, effective_threshold)
+        ComparisonResult.new(results, criterion: display_criterion)
+      end
+
+      # Wrap a block with VCR cassette recording/playback
+      #
+      # @param name [String] Cassette name
+      # @param record [Symbol] Recording mode (:new_episodes, :none, :all)
+      # @yield Block to execute with cassette
+      #
+      # @example
+      #   with_qualspec_cassette("my_test") do
+      #     result = qualspec_evaluate(response, "is helpful")
+      #     expect(result).to be_passing
+      #   end
+      #
+      def with_qualspec_cassette(name, record: nil, &block)
+        record_mode = record || Qualspec::RSpec.configuration.record_mode
+
+        # Configure VCR with RSpec cassette directory
+        Qualspec::Recorder.setup(
+          cassette_dir: Qualspec::RSpec.configuration.vcr_cassette_dir
+        )
+
+        case record_mode
+        when :none
+          Qualspec::Recorder.playback(name, &block)
+        else
+          VCR.use_cassette(name, record: record_mode, &block)
+        end
+      end
+
+      # Helper to skip test if qualspec API is unavailable
+      def skip_without_qualspec_api
+        Qualspec.client.chat(
+          model: Qualspec.configuration.judge_model,
+          messages: [{ role: 'user', content: 'test' }],
+          json_mode: false
+        )
+      rescue Qualspec::Client::RequestError => e
+        skip "Qualspec API unavailable: #{e.message}"
+      end
+
+      private
+
+      def resolve_criterion(criterion, rubric)
+        if rubric
+          rubric_obj = rubric.is_a?(Symbol) ? Qualspec::Rubric.find(rubric) : rubric
+          criterion_text = rubric_obj.criteria.join("\n")
+          display_criterion = rubric_obj.criteria.join('; ')
+          [criterion_text, display_criterion]
+        else
+          [criterion, criterion]
+        end
+      end
+
+      def wrap_comparison_results(evaluations, responses, display_criterion, threshold)
+        evaluations.transform_keys(&:to_sym).each_with_object({}) do |(name, eval), hash|
+          hash[name] = EvaluationResult.new(
+            eval,
+            criterion: display_criterion,
+            response: responses[name] || responses[name.to_s],
+            threshold: threshold
+          )
+        end
+      end
+    end
+  end
+end

data/lib/qualspec/rspec/matchers.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require 'rspec/expectations'
+
+module Qualspec
+  module RSpec
+    # Custom RSpec matchers for qualspec evaluations
+    #
+    # These matchers work with EvaluationResult objects returned by qualspec_evaluate
+    #
+    # @example
+    #   result = qualspec_evaluate(response, "is helpful")
+    #   expect(result).to be_passing
+    #   expect(result).to have_score_above(7)
+    #
+    module Matchers
+      extend ::RSpec::Matchers::DSL
+
+      # Matcher for checking if evaluation passed
+      #
+      # @example
+      #   expect(result).to be_passing
+      #
+      matcher :be_passing do
+        match do |result|
+          result.respond_to?(:passing?) && result.passing?
+        end
+
+        failure_message do |result|
+          if result.respond_to?(:failure_message)
+            result.failure_message
+          else
+            "expected #{result.inspect} to be passing"
+          end
+        end
+
+        failure_message_when_negated do |result|
+          if result.respond_to?(:failure_message_when_negated)
+            result.failure_message_when_negated
+          else
+            "expected #{result.inspect} not to be passing"
+          end
+        end
+      end
+
+      # Matcher for checking if evaluation failed
+      #
+      # @example
+      #   expect(result).to be_failing
+      #
+      matcher :be_failing do
+        match do |result|
+          result.respond_to?(:failing?) && result.failing?
+        end
+
+        failure_message do |result|
+          "expected evaluation to fail, but it passed with score #{result.score}/10"
+        end
+
+        failure_message_when_negated do |result|
+          "expected evaluation not to fail, but it failed with score #{result.score}/10"
+        end
+      end
+
+      # Matcher for checking exact score
+      #
+      # @example
+      #   expect(result).to have_score(10)
+      #
+      matcher :have_score do |expected|
+        match do |result|
+          result.respond_to?(:score) && result.score == expected
+        end
+
+        failure_message do |result|
+          "expected score to be #{expected}, but was #{result.score}"
+        end
+      end
+
+      # Matcher for checking score is above threshold
+      #
+      # @example
+      #   expect(result).to have_score_above(7)
+      #
+      matcher :have_score_above do |threshold|
+        match do |result|
+          result.respond_to?(:score) && result.score > threshold
+        end
+
+        failure_message do |result|
+          "expected score > #{threshold}, but was #{result.score}"
+        end
+      end
+
+      # Matcher for checking score is at or above threshold
+      #
+      # @example
+      #   expect(result).to have_score_at_least(7)
+      #
+      matcher :have_score_at_least do |threshold|
+        match do |result|
+          result.respond_to?(:score) && result.score >= threshold
+        end
+
+        failure_message do |result|
+          "expected score >= #{threshold}, but was #{result.score}"
+        end
+      end
+
+      # Matcher for checking score is below threshold
+      #
+      # @example
+      #   expect(result).to have_score_below(5)
+      #
+      matcher :have_score_below do |threshold|
+        match do |result|
+          result.respond_to?(:score) && result.score < threshold
+        end
+
+        failure_message do |result|
+          "expected score < #{threshold}, but was #{result.score}"
+        end
+      end
+
+      # Matcher for comparison winner
+      #
+      # @example
+      #   expect(comparison).to have_winner(:claude)
+      #
+      matcher :have_winner do |expected_winner|
+        match do |result|
+          result.respond_to?(:winner?) && result.winner?(expected_winner)
+        end
+
+        failure_message do |result|
+          "expected winner to be #{expected_winner}, but was #{result.winner}"
+        end
+      end
+
+      # Matcher for comparison tie
+      #
+      # @example
+      #   expect(comparison).to be_a_tie
+      #
+      matcher :be_a_tie do
+        match do |result|
+          result.respond_to?(:tie?) && result.tie?
+        end
+
+        failure_message do |result|
+          "expected a tie, but winner was #{result.winner}"
+        end
+      end
+    end
+  end
+end
+
+# Auto-include matchers when RSpec is loaded
+if defined?(RSpec) && RSpec.respond_to?(:configure)
+  RSpec.configure do |config|
+    config.include Qualspec::RSpec::Matchers
+  end
+end

data/lib/qualspec/rspec.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+# RSpec integration for Qualspec
+#
+# Add to your spec_helper.rb:
+#
+#   require "qualspec/rspec"
+#
+#   RSpec.configure do |config|
+#     config.include Qualspec::RSpec::Helpers
+#   end
+#
+#   # Optional: Configure qualspec-specific settings
+#   Qualspec::RSpec.configure do |config|
+#     config.default_threshold = 7
+#     config.vcr_cassette_dir = "spec/cassettes/qualspec"
+#   end
+#
+# Then in your specs:
+#
+#   describe "MyAgent" do
+#     it "responds helpfully" do
+#       response = my_agent.call("Hello")
+#
+#       result = qualspec_evaluate(response, "responds in a friendly manner")
+#       expect(result).to be_passing
+#       expect(result.score).to be >= 8
+#     end
+#
+#     it "uses tools correctly" do
+#       response = my_agent.call("What's the weather?")
+#
+#       result = qualspec_evaluate(response, rubric: :tool_calling)
+#       expect(result).to be_passing
+#     end
+#   end
+#
+
+require 'qualspec'
+require_relative 'rspec/configuration'
+require_relative 'rspec/evaluation_result'
+require_relative 'rspec/helpers'
+require_relative 'rspec/matchers'
+
+module Qualspec
+  module RSpec
+    class << self
+      # Setup RSpec integration with sensible defaults
+      #
+      # @example
+      #   Qualspec::RSpec.setup!
+      #
+      def setup!
+        # Load builtin rubrics if configured
+        Qualspec::BuiltinRubrics.load! if configuration.load_builtins
+
+        # Configure RSpec if available
+        return unless defined?(::RSpec) && ::RSpec.respond_to?(:configure)
+
+        ::RSpec.configure do |config|
+          config.include Qualspec::RSpec::Helpers
+        end
+      end
+    end
+  end
+end

data/lib/qualspec/rubric.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Qualspec
+  class Rubric
+    attr_reader :name, :criteria
+
+    def initialize(name, &block)
+      @name = name
+      @criteria = []
+      instance_eval(&block) if block_given? # rubocop:disable Style/EvalWithLocation -- DSL pattern requires eval
+    end
+
+    def criterion(description)
+      @criteria << description
+    end
+
+    def to_s
+      @criteria.join("\n")
+    end
+
+    class << self
+      def registry
+        @registry ||= {}
+      end
+
+      def define(name, &block)
+        registry[name.to_sym] = new(name, &block)
+      end
+
+      def find(name)
+        registry[name.to_sym] || raise(Error, "Rubric '#{name}' not found")
+      end
+
+      def defined?(name)
+        registry.key?(name.to_sym)
+      end
+
+      def clear!
+        @registry = {}
+      end
+    end
+  end
+end