ruby_llm-contract 0.4.5 → 0.5.0

data/lib/ruby_llm/contract/eval/case_scorer.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      class CaseScorer
+        include TraitEvaluator
+        include ContractDetailBuilder
+
+        def initialize(step:, expectation_evaluator: ExpectationEvaluator.new)
+          @step = step
+          @expectation_evaluator = expectation_evaluator
+        end
+
+        def call(test_case:, step_result:)
+          return contract_failure(step_result) unless step_result.ok?
+
+          if test_case.evaluator
+            evaluate_with_custom(test_case, step_result)
+          elsif test_case.expected_traits
+            evaluate_traits(step_result, test_case)
+          elsif !test_case.expected.nil?
+            evaluate_expected(test_case, step_result)
+          else
+            evaluate_contract_only
+          end
+        end
+
+        private
+
+        def evaluate_expected(test_case, step_result)
+          @expectation_evaluator.call(
+            output: step_result.parsed_output,
+            expected: test_case.expected,
+            input: test_case.input
+          )
+        end
+
+        def evaluate_with_custom(test_case, step_result)
+          wrapped_custom_evaluator(test_case).call(
+            output: step_result.parsed_output,
+            expected: test_case.expected,
+            input: test_case.input
+          )
+        end
+
+        def wrapped_custom_evaluator(test_case)
+          evaluator = test_case.evaluator
+          evaluator.is_a?(::Proc) ? Evaluator::ProcEvaluator.new(evaluator) : evaluator
+        end
+
+        def evaluate_contract_only
+          EvaluationResult.new(score: 1.0, passed: true, details: build_contract_details)
+        end
+
+        def contract_failure(step_result)
+          EvaluationResult.new(
+            score: 0.0,
+            passed: false,
+            details: "step failed: #{step_result.status} — #{step_result.validation_errors.join(", ")}"
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/evaluator/exact.rb

@@ -4,14 +4,16 @@ module RubyLLM
   module Contract
     module Eval
       module Evaluator
+        # Compares output to expected using Ruby equality semantics.
         class Exact
           def call(output:, expected:, input: nil) # rubocop:disable Lint/UnusedMethodArgument
-            if output == expected
-              EvaluationResult.new(score: 1.0, passed: true, details: "exact match")
-            else
-              EvaluationResult.new(score: 0.0, passed: false,
-                                   details: "expected #{expected.inspect}, got #{output.inspect}")
-            end
+            return EvaluationResult.new(score: 1.0, passed: true, details: "exact match") if output == expected
+
+            EvaluationResult.new(
+              score: 0.0,
+              passed: false,
+              details: "expected #{expected.inspect}, got #{output.inspect}"
+            )
           end
         end
       end

data/lib/ruby_llm/contract/eval/evaluator/proc_evaluator.rb

@@ -4,24 +4,36 @@ module RubyLLM
   module Contract
     module Eval
       module Evaluator
+        # Adapts custom Ruby callables to the EvaluationResult contract.
         class ProcEvaluator
           def initialize(callable)
             @callable = callable
           end
 
           def call(output:, expected: nil, input: nil) # rubocop:disable Lint/UnusedMethodArgument,Metrics
-            result = if @callable.arity == 2 || (@callable.arity.negative? && @callable.parameters.length >= 2)
-                       @callable.call(output, input)
-                     else
-                       @callable.call(output)
-                     end
+            result = invoke_callable(output, input)
+            warn_nil_result if result.nil?
+            build_evaluation_result(result)
+          end
 
-            if result.nil?
-              warn "[ruby_llm-contract] verify/evaluator proc returned nil. " \
-                   "This usually means a key mismatch (string vs symbol). " \
-                   "Output keys are always symbols."
-            end
+          private
+
+          def invoke_callable(output, input)
+            callable_accepts_input? ? @callable.call(output, input) : @callable.call(output)
+          end
+
+          def callable_accepts_input?
+            arity = @callable.arity
+            arity == 2 || (arity.negative? && @callable.parameters.length >= 2)
+          end
+
+          def warn_nil_result
+            warn "[ruby_llm-contract] verify/evaluator proc returned nil. " \
+                 "This usually means a key mismatch (string vs symbol). " \
+                 "Output keys are always symbols."
+          end
 
+          def build_evaluation_result(result)
             case result
             when true
               EvaluationResult.new(score: 1.0, passed: true, details: "passed")

data/lib/ruby_llm/contract/eval/evaluator/regex.rb

@@ -4,21 +4,24 @@ module RubyLLM
   module Contract
     module Eval
       module Evaluator
+        # Matches a regex against the flattened textual representation of output.
         class Regex
           def initialize(pattern)
             @pattern = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(pattern)
           end
 
           def call(output:, expected: nil, input: nil) # rubocop:disable Lint/UnusedMethodArgument
-            text = output.is_a?(Hash) ? output.values.join(" ") : output.to_s
+            pattern = @pattern.inspect
+            details = text_for(output).match?(@pattern) ? "matches #{pattern}" : "does not match #{pattern}"
+            passed = details.start_with?("matches")
 
-            if text.match?(@pattern)
-              EvaluationResult.new(score: 1.0, passed: true,
-                                   details: "matches #{@pattern.inspect}")
-            else
-              EvaluationResult.new(score: 0.0, passed: false,
-                                   details: "does not match #{@pattern.inspect}")
-            end
+            EvaluationResult.new(score: passed ? 1.0 : 0.0, passed: passed, details: details)
+          end
+
+          private
+
+          def text_for(output)
+            output.is_a?(Hash) ? output.values.join(" ") : output.to_s
           end
         end
       end

data/lib/ruby_llm/contract/eval/expectation_evaluator.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      class ExpectationEvaluator
+        def call(output:, expected:, input:)
+          evaluator_for(expected).call(output: output, expected: expected, input: input)
+        end
+
+        private
+
+        def evaluator_for(expected)
+          case expected
+          when Hash
+            Evaluator::JsonIncludes.new
+          when ::Regexp
+            Evaluator::Regex.new(expected)
+          else
+            Evaluator::Exact.new
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/prompt_diff.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module RubyLLM
+  module Contract
+    module Eval
+      class PromptDiff
+        extend Forwardable
+
+        attr_reader :candidate_report, :baseline_report
+        def_delegators :@diff, :improvements, :regressions, :score_delta, :removed_passing_cases
+        def_delegators :@comparator, :safe_to_switch?, :case_names_match?, :cases_comparable?, :mismatched_cases,
+                       :input_mismatches, :expected_mismatches, :score_regressions, :candidate_score, :baseline_score,
+                       :baseline_empty?, :candidate_empty?
+        def_delegators :@presenter, :print_summary
+
+        def initialize(candidate:, baseline:)
+          @candidate_report = candidate
+          @baseline_report = baseline
+          serializer = PromptDiffSerializer.new
+          candidate_cases = serializer.call(candidate)
+          baseline_cases = serializer.call(baseline)
+          @diff = BaselineDiff.new(
+            baseline_cases: baseline_cases,
+            current_cases: candidate_cases
+          )
+          @comparator = PromptDiffComparator.new(
+            candidate_cases: candidate_cases,
+            baseline_cases: baseline_cases,
+            diff: @diff
+          )
+          @presenter = PromptDiffPresenter.new(prompt_diff: self, comparator: @comparator)
+          freeze
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/prompt_diff_comparator.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      # Encapsulates the safety and mismatch rules for prompt A/B comparison.
+      class PromptDiffComparator
+        def initialize(candidate_cases:, baseline_cases:, diff:)
+          @candidate_cases = candidate_cases
+          @baseline_cases = baseline_cases
+          @diff = diff
+          @baseline_case_index = baseline_cases.to_h { |case_data| [case_data[:name], case_data] }
+        end
+
+        def safe_to_switch?
+          return false if empty_comparison?
+          return false unless cases_comparable?
+          return false if score_regressions.any?
+
+          !@diff.regressed?
+        end
+
+        def case_names_match?
+          case_names(@baseline_cases) == case_names(@candidate_cases)
+        end
+
+        def cases_comparable?
+          case_signatures(@baseline_cases) == case_signatures(@candidate_cases)
+        end
+
+        def mismatched_cases
+          baseline_names = case_names(@baseline_cases)
+          candidate_names = case_names(@candidate_cases)
+
+          {
+            only_in_baseline: baseline_names - candidate_names,
+            only_in_candidate: candidate_names - baseline_names
+          }
+        end
+
+        def input_mismatches
+          attribute_mismatches(:input, :baseline_input, :candidate_input)
+        end
+
+        def expected_mismatches
+          attribute_mismatches(:expected, :baseline_expected, :candidate_expected)
+        end
+
+        def score_regressions
+          @candidate_cases.filter_map do |candidate_case|
+            baseline_case = @baseline_case_index[candidate_case[:name]]
+            next unless baseline_case
+
+            baseline_score = baseline_case[:score]
+            candidate_score = candidate_case[:score]
+            next unless candidate_score < baseline_score
+
+            {
+              case: candidate_case[:name],
+              baseline_score: baseline_score,
+              candidate_score: candidate_score,
+              delta: (candidate_score - baseline_score).round(4)
+            }
+          end
+        end
+
+        def candidate_score
+          @diff.current_score
+        end
+
+        def baseline_score
+          @diff.baseline_score
+        end
+
+        def candidate_empty?
+          @candidate_cases.empty?
+        end
+
+        def baseline_empty?
+          @baseline_cases.empty?
+        end
+
+        def empty_comparison?
+          baseline_empty? || candidate_empty?
+        end
+
+        private
+
+        def case_names(cases)
+          cases.map { |case_data| case_data[:name] }.sort
+        end
+
+        def case_signatures(cases)
+          cases.map { |case_data| [case_data[:name], case_data[:input], case_data[:expected]] }.sort_by(&:first)
+        end
+
+        def attribute_mismatches(attribute, baseline_key, candidate_key)
+          @candidate_cases.filter_map do |candidate_case|
+            baseline_case = @baseline_case_index[candidate_case[:name]]
+            next unless baseline_case
+
+            baseline_value = baseline_case[attribute]
+            candidate_value = candidate_case[attribute]
+            next if baseline_value == candidate_value
+
+            {
+              case: candidate_case[:name],
+              baseline_key => baseline_value,
+              candidate_key => candidate_value
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/prompt_diff_presenter.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      # Renders a prompt diff as a readable console summary.
+      class PromptDiffPresenter
+        VARIANT_LABEL_WIDTH = 12
+        TABLE_WIDTH = 26
+        CASE_SET_MISMATCH_TITLE = "  Case set mismatch (safe_to_switch? = NO):"
+        INPUT_MISMATCH_TITLE = "  Input mismatch (safe_to_switch? = NO):"
+        EXPECTED_MISMATCH_TITLE = "  Expected mismatch (safe_to_switch? = NO):"
+        REGRESSIONS_TITLE = "  Regressions (PASS -> FAIL):"
+        SCORE_DROPS_TITLE = "  Score drops:"
+        IMPROVEMENTS_TITLE = "  Improvements:"
+        REMOVED_PASSING_TITLE = "  Removed (were passing in baseline):"
+
+        def initialize(prompt_diff:, comparator:)
+          @prompt_diff = prompt_diff
+          @comparator = comparator
+        end
+
+        def print_summary(io = $stdout)
+          print_header(io)
+          print_warning(io, "one side has no evaluated cases (all skipped?)") if @comparator.empty_comparison?
+          print_case_set_mismatch(io)
+          print_formatted_section(io, INPUT_MISMATCH_TITLE, @comparator.input_mismatches) do |mismatch|
+            "#{mismatch[:case]}: inputs differ between candidate and baseline"
+          end
+          print_formatted_section(io, EXPECTED_MISMATCH_TITLE, @comparator.expected_mismatches) do |mismatch|
+            "#{mismatch[:case]}: expected values differ between candidate and baseline"
+          end
+          print_formatted_section(io, REGRESSIONS_TITLE, @prompt_diff.regressions) do |regression|
+            "#{regression[:case]}: was PASS, now FAIL -- #{regression[:detail]}"
+          end
+          print_formatted_section(io, SCORE_DROPS_TITLE, @comparator.score_regressions) do |regression|
+            "#{regression[:case]}: #{regression[:baseline_score]} -> #{regression[:candidate_score]} (#{regression[:delta]})"
+          end
+          print_formatted_section(io, IMPROVEMENTS_TITLE, @prompt_diff.improvements) do |improvement|
+            "#{improvement[:case]}: was FAIL, now PASS"
+          end
+          print_formatted_section(io, REMOVED_PASSING_TITLE, @prompt_diff.removed_passing_cases, &:to_s)
+          io.puts "  Safe to switch: #{@comparator.safe_to_switch? ? "YES" : "NO"}"
+        end
+
+        private
+
+        def print_header(io)
+          lines = [
+            "Prompt A/B comparison",
+            nil,
+            format("  %-#{VARIANT_LABEL_WIDTH}s  Score", "Variant"),
+            "  #{"-" * TABLE_WIDTH}",
+            format("  %-#{VARIANT_LABEL_WIDTH}s  %.2f", "Candidate", @comparator.candidate_score),
+            format("  %-#{VARIANT_LABEL_WIDTH}s  %.2f", "Baseline", @comparator.baseline_score),
+            nil,
+            "  Score delta: #{format_delta(@prompt_diff.score_delta)}",
+            nil
+          ]
+          emit_lines(io, lines)
+        end
+
+        def print_warning(io, message)
+          emit_lines(io, ["  WARNING: #{message}", nil])
+        end
+
+        def print_case_set_mismatch(io)
+          return if @comparator.case_names_match?
+
+          mismatches = @comparator.mismatched_cases
+          lines = mismatches[:only_in_baseline].map { |name| "only in baseline: #{name}" } +
+            mismatches[:only_in_candidate].map { |name| "only in candidate: #{name}" }
+          emit_section(io, CASE_SET_MISMATCH_TITLE, lines)
+        end
+
+        def print_formatted_section(io, title, collection)
+          return if collection.empty?
+
+          lines = collection.map { |entry| yield(entry) }
+          emit_section(io, title, lines)
+        end
+
+        def emit_section(io, title, lines)
+          emit_lines(io, [title, *lines.map { |line| "    #{line}" }, nil])
+        end
+
+        def emit_lines(io, lines)
+          lines.each do |line|
+            line.nil? ? io.puts : io.puts(line)
+          end
+        end
+
+        def format_delta(delta)
+          delta >= 0 ? "+#{delta}" : delta.to_s
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/prompt_diff_serializer.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      # Normalizes report results into comparable prompt-diff case hashes.
+      class PromptDiffSerializer
+        def call(report)
+          report.results.reject { |result| result.step_status == :skipped }.map do |result|
+            {
+              name: result.name,
+              input: result.input,
+              expected: result.expected,
+              passed: result.passed?,
+              score: result.score,
+              details: result.details
+            }
+          end
+        end
+      end
+    end
+  end
+end