ruby_llm-contract 0.4.5 → 0.5.0

data/lib/ruby_llm/contract/eval/runner.rb

@@ -4,8 +4,6 @@ module RubyLLM
   module Contract
     module Eval
       class Runner
-        include TraitEvaluator
-        include ContractDetailBuilder
         include Concerns::ContextHelpers
 
         def self.run(step:, dataset:, context: {}, concurrency: nil)
@@ -20,17 +18,35 @@ module RubyLLM
         end
 
         def run
-          results = if @concurrency && @concurrency > 1
-                      run_concurrent
-                    else
-                      @dataset.cases.map { |test_case| evaluate_case(test_case) }
-                    end
-          step_name = @step.respond_to?(:name) ? @step.name : @step.to_s
-          Report.new(dataset_name: @dataset.name, results: results, step_name: step_name)
+          Report.new(
+            dataset_name: @dataset.name,
+            results: collected_results,
+            step_name: step_name
+          )
         end
 
         private
 
+        def collected_results
+          concurrent? ? run_concurrent : run_serial
+        end
+
+        def run_serial
+          @dataset.cases.map { |test_case| case_executor.call(test_case: test_case, context: @context) }
+        end
+
+        def concurrent?
+          @concurrency && @concurrency > 1
+        end
+
+        def step_name
+          @step.respond_to?(:name) ? @step.name : @step.to_s
+        end
+
+        def case_executor
+          @case_executor ||= CaseExecutor.new(step: @step)
+        end
+
         def run_concurrent
           require "concurrent"
           pool = Concurrent::FixedThreadPool.new(@concurrency)
@@ -39,10 +55,10 @@ module RubyLLM
           # gets a single-response adapter with its own response (by index).
           per_case_contexts = build_per_case_contexts
 
-          futures = @dataset.cases.each_with_index.map do |test_case, i|
-            ctx = per_case_contexts[i]
+          futures = @dataset.cases.each_with_index.map do |test_case, index|
+            case_context = per_case_contexts[index]
             Concurrent::Future.execute(executor: pool) do
-              evaluate_case_with_context(test_case, ctx)
+              case_executor.call(test_case: test_case, context: case_context)
             end
           end
           futures.map(&:value!)
@@ -55,10 +71,10 @@ module RubyLLM
           adapter = @context[:adapter]
           responses = adapter.respond_to?(:responses_array) ? adapter.responses_array : nil
 
-          @dataset.cases.each_with_index.map do |_, i|
+          @dataset.cases.each_with_index.map do |_, index|
             if responses
               # Give each case its own single-response adapter
-              response = responses[i] || responses.last
+              response = responses[index] || responses.last
               per_case_adapter = Adapters::Test.new(response: response)
               @context.merge(adapter: per_case_adapter)
             else
@@ -66,199 +82,6 @@ module RubyLLM
             end
           end
         end
-
-        def evaluate_case_with_context(test_case, context)
-          run_result = @step.run(test_case.input, context: context)
-          step_result = normalize_result(run_result)
-          eval_result = dispatch_evaluation(step_result, test_case)
-
-          result = build_case_result(test_case, step_result, eval_result)
-
-          if test_case.respond_to?(:step_expectations) && test_case.step_expectations &&
-             run_result.respond_to?(:outputs_by_step)
-            evaluate_step_expectations(result, run_result.outputs_by_step, test_case.step_expectations)
-          else
-            result
-          end
-        rescue RubyLLM::Contract::Error => e
-          raise unless e.message.include?("No adapter configured")
-
-          skipped_result(test_case, e.message)
-        end
-
-        def evaluate_case(test_case)
-          run_result = @step.run(test_case.input, context: @context)
-          step_result = normalize_result(run_result)
-          eval_result = dispatch_evaluation(step_result, test_case)
-
-          result = build_case_result(test_case, step_result, eval_result)
-
-          # Pipeline per-step evaluation
-          if test_case.respond_to?(:step_expectations) && test_case.step_expectations &&
-             run_result.respond_to?(:outputs_by_step)
-            evaluate_step_expectations(result, run_result.outputs_by_step, test_case.step_expectations)
-          else
-            result
-          end
-        rescue RubyLLM::Contract::Error => e
-          raise unless e.message.include?("No adapter configured")
-
-          skipped_result(test_case, e.message)
-        end
-
-        def build_case_result(test_case, step_result, eval_result)
-          trace = step_result.respond_to?(:trace) ? step_result.trace : nil
-          CaseResult.new(
-            name: test_case.name,
-            input: test_case.input,
-            output: step_result.parsed_output,
-            expected: test_case.expected,
-            step_status: step_result.status,
-            score: eval_result.score,
-            passed: eval_result.passed,
-            label: eval_result.label,
-            details: eval_result.details,
-            duration_ms: extract_latency(trace),
-            cost: extract_cost(trace)
-          )
-        end
-
-        def extract_latency(trace)
-          return nil unless trace
-
-          # Pipeline::Trace uses total_latency_ms, Step::Trace uses latency_ms
-          if trace.respond_to?(:total_latency_ms)
-            trace.total_latency_ms
-          else
-            trace[:latency_ms]
-          end
-        end
-
-        def extract_cost(trace)
-          return nil unless trace
-
-          # Pipeline::Trace uses total_cost, Step::Trace uses cost
-          if trace.respond_to?(:total_cost)
-            trace.total_cost
-          else
-            trace[:cost]
-          end
-        end
-
-        def dispatch_evaluation(step_result, test_case)
-          return contract_failure(step_result) unless step_result.ok?
-
-          if test_case.evaluator
-            evaluate_with_custom(step_result, test_case)
-          elsif test_case.expected_traits
-            evaluate_traits(step_result, test_case)
-          elsif !test_case.expected.nil?
-            evaluate_expected(step_result, test_case)
-          else
-            evaluate_contract_only
-          end
-        end
-
-        def normalize_result(result)
-          return result if result.respond_to?(:parsed_output)
-
-          normalize_pipeline_result(result)
-        end
-
-        def normalize_pipeline_result(result)
-          last_result = result.step_results&.last&.dig(:result)
-          is_ok = result.ok?
-          pipeline_trace = result.respond_to?(:trace) ? result.trace : nil
-
-          PipelineResultAdapter.new(
-            status: result.status,
-            ok_flag: is_ok,
-            parsed_output: is_ok ? result.outputs_by_step.values.last : nil,
-            validation_errors: last_result.respond_to?(:validation_errors) ? last_result.validation_errors : [],
-            trace: pipeline_trace || (last_result.respond_to?(:trace) ? last_result.trace : {})
-          )
-        end
-
-        def evaluate_expected(step_result, test_case)
-          dispatch_expected_evaluator(
-            output: step_result.parsed_output,
-            expected: test_case.expected,
-            input: test_case.input
-          )
-        end
-
-        def dispatch_expected_evaluator(output:, expected:, input:)
-          if expected.is_a?(Hash)
-            Evaluator::JsonIncludes.new.call(output: output, expected: expected, input: input)
-          elsif expected.is_a?(::Regexp)
-            Evaluator::Regex.new(expected).call(output: output, input: input)
-          else
-            Evaluator::Exact.new.call(output: output, expected: expected, input: input)
-          end
-        end
-
-        def evaluate_with_custom(step_result, test_case)
-          evaluator = test_case.evaluator
-          evaluator = Evaluator::ProcEvaluator.new(evaluator) if evaluator.is_a?(::Proc)
-          evaluator.call(output: step_result.parsed_output, expected: test_case.expected, input: test_case.input)
-        end
-
-        def evaluate_contract_only
-          EvaluationResult.new(score: 1.0, passed: true, details: build_contract_details)
-        end
-
-        def contract_failure(step_result)
-          EvaluationResult.new(
-            score: 0.0, passed: false,
-            details: "step failed: #{step_result.status} — #{step_result.validation_errors.join(", ")}"
-          )
-        end
-
-        def evaluate_step_expectations(result, outputs_by_step, expectations)
-          step_results = {}
-          all_passed = true
-
-          expectations.each do |step_alias, expected|
-            output = outputs_by_step[step_alias]
-            if output.nil?
-              step_results[step_alias] = { passed: false, details: "step not executed" }
-              all_passed = false
-            else
-              eval_res = dispatch_expected_evaluator(output: output, expected: expected, input: nil)
-              step_results[step_alias] = { passed: eval_res.passed, score: eval_res.score, details: eval_res.details }
-              all_passed = false unless eval_res.passed
-            end
-          end
-
-          # Rebuild CaseResult with step_results metadata
-          failed_steps = step_results.select { |_, v| !v[:passed] }
-          failure_details = failed_steps.map { |k, v| "#{k}: #{v[:details]}" }.join("; ")
-
-          CaseResult.new(
-            name: result.name, input: result.input, output: result.output,
-            expected: result.expected,
-            step_status: all_passed ? result.step_status : :step_expectation_failed,
-            score: all_passed ? result.score : 0.0,
-            passed: result.passed? && all_passed,
-            label: all_passed ? result.label : "FAIL",
-            details: all_passed ? result.details : "step expectations failed: #{failure_details}",
-            duration_ms: result.duration_ms, cost: result.cost
-          )
-        end
-
-        def skipped_result(test_case, reason)
-          CaseResult.new(
-            name: test_case.name,
-            input: test_case.input,
-            output: nil,
-            expected: test_case.expected,
-            step_status: :skipped,
-            score: 0.0,
-            passed: false,
-            label: "SKIP",
-            details: "skipped: #{reason}"
-          )
-        end
       end
     end
   end

data/lib/ruby_llm/contract/eval/step_expectation_applier.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      class StepExpectationApplier
+        def initialize(expectation_evaluator: ExpectationEvaluator.new)
+          @expectation_evaluator = expectation_evaluator
+        end
+
+        def call(result:, run_result:, test_case:)
+          return result unless applicable?(test_case, run_result)
+
+          expectation_results = evaluate_expectations(run_result.outputs_by_step, test_case.step_expectations)
+          return result if expectation_results.values.all? { |entry| entry[:passed] }
+
+          rebuild_result(result, failure_details_for(expectation_results))
+        end
+
+        private
+
+        def applicable?(test_case, run_result)
+          test_case.respond_to?(:step_expectations) &&
+            test_case.step_expectations &&
+            run_result.respond_to?(:outputs_by_step)
+        end
+
+        def evaluate_expectations(outputs_by_step, expectations)
+          expectations.each_with_object({}) do |(step_alias, expected), results|
+            output = outputs_by_step[step_alias]
+            results[step_alias] = evaluate_single_expectation(output, expected)
+          end
+        end
+
+        def evaluate_single_expectation(output, expected)
+          return { passed: false, details: "step not executed" } if output.nil?
+
+          evaluation = @expectation_evaluator.call(output: output, expected: expected, input: nil)
+          { passed: evaluation.passed, details: evaluation.details }
+        end
+
+        def failure_details_for(expectation_results)
+          expectation_results
+            .select { |_, entry| !entry[:passed] }
+            .map { |step_alias, entry| "#{step_alias}: #{entry[:details]}" }
+            .join("; ")
+        end
+
+        def rebuild_result(result, failure_details)
+          CaseResult.new(
+            name: result.name,
+            input: result.input,
+            output: result.output,
+            expected: result.expected,
+            step_status: :step_expectation_failed,
+            score: 0.0,
+            passed: false,
+            label: "FAIL",
+            details: "step expectations failed: #{failure_details}",
+            duration_ms: result.duration_ms,
+            cost: result.cost
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/step_result_normalizer.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      class StepResultNormalizer
+        def call(result)
+          return result if result.respond_to?(:parsed_output)
+
+          normalize_pipeline_result(result)
+        end
+
+        private
+
+        def normalize_pipeline_result(result)
+          last_result = result.step_results&.last&.dig(:result)
+          successful = result.ok?
+          trace = result.respond_to?(:trace) ? result.trace : nil
+
+          PipelineResultAdapter.new(
+            status: result.status,
+            ok_flag: successful,
+            parsed_output: successful ? result.outputs_by_step.values.last : nil,
+            validation_errors: validation_errors_for(last_result),
+            trace: trace || trace_for(last_result)
+          )
+        end
+
+        def validation_errors_for(result)
+          result.respond_to?(:validation_errors) ? result.validation_errors : []
+        end
+
+        def trace_for(result)
+          result.respond_to?(:trace) ? result.trace : {}
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval.rb

@@ -8,11 +8,24 @@ require_relative "eval/evaluator/json_includes"
 require_relative "eval/evaluator/proc_evaluator"
 require_relative "eval/dataset"
 require_relative "eval/pipeline_result_adapter"
+require_relative "eval/expectation_evaluator"
 require_relative "eval/trait_evaluator"
 require_relative "eval/contract_detail_builder"
+require_relative "eval/case_scorer"
+require_relative "eval/case_result_builder"
+require_relative "eval/step_result_normalizer"
+require_relative "eval/step_expectation_applier"
+require_relative "eval/case_executor"
 require_relative "eval/runner"
+require_relative "eval/report_stats"
+require_relative "eval/report_presenter"
+require_relative "eval/report_storage"
 require_relative "eval/report"
 require_relative "eval/eval_definition"
 require_relative "eval/model_comparison"
 require_relative "eval/baseline_diff"
+require_relative "eval/prompt_diff_serializer"
+require_relative "eval/prompt_diff_comparator"
+require_relative "eval/prompt_diff_presenter"
+require_relative "eval/prompt_diff"
 require_relative "eval/eval_history"

data/lib/ruby_llm/contract/pipeline/base.rb

@@ -25,7 +25,16 @@ module RubyLLM
 
           # Internal mutable steps list for registration
           def steps_registry
-            @steps_registry ||= []
+            @steps_registry ||= begin
+              inherited_steps =
+                if superclass.respond_to?(:steps_registry, true)
+                  superclass.send(:steps_registry).map(&:dup)
+                else
+                  []
+                end
+
+              inherited_steps
+            end
           end
 
           def token_budget(limit = nil)

data/lib/ruby_llm/contract/rspec/pass_eval.rb

@@ -68,15 +68,28 @@ RSpec::Matchers.define :pass_eval do |eval_name|
     @check_regressions = true
   end
 
+  chain :compared_with do |other_step|
+    @comparison_step = other_step
+    @check_regressions = true # compared_with implies regression check
+  end
+
   match do |step_or_pipeline|
     @eval_name = eval_name
     @context ||= {}
     @minimum_score ||= nil
     @maximum_cost ||= nil
     @check_regressions ||= false
+    @comparison_step ||= nil
     @error = nil
     @diff = nil
-    @report = step_or_pipeline.run_eval(eval_name, context: @context)
+    @prompt_diff = nil
+
+    if @comparison_step && @check_regressions
+      @prompt_diff = step_or_pipeline.compare_with(@comparison_step, eval: eval_name, context: @context)
+      @report = @prompt_diff.candidate_report
+    else
+      @report = step_or_pipeline.run_eval(eval_name, context: @context)
+    end
 
     score_ok = if @minimum_score
                  @report.score >= @minimum_score
@@ -86,7 +99,9 @@ RSpec::Matchers.define :pass_eval do |eval_name|
 
     cost_ok = @maximum_cost ? @report.total_cost <= @maximum_cost : true
 
-    regression_ok = if @check_regressions && @report.baseline_exists?
+    regression_ok = if @prompt_diff
+                      @prompt_diff.safe_to_switch?
+                    elsif @check_regressions && @report.baseline_exists?
                       @diff = @report.compare_with_baseline
                       !@diff.regressed?
                     else
@@ -100,11 +115,67 @@ RSpec::Matchers.define :pass_eval do |eval_name|
   end
 
   failure_message do
+    if @prompt_diff && !@prompt_diff.safe_to_switch?
+      msg = "expected #{@eval_name} eval to be safe to switch from baseline prompt\n"
+
+      # Check empty sides first — most fundamental problem
+      bl_empty = @prompt_diff.baseline_empty?
+      cd_empty = @prompt_diff.candidate_empty?
+      if bl_empty || cd_empty
+        msg += "  One side has no evaluated cases (all skipped or no adapter?)\n"
+        if sample_response_only_compare?
+          msg += "  compare_with ignores sample_response; pass model: or with_context(adapter: ...)\n"
+        end
+        msg += "  Candidate score: #{@prompt_diff.candidate_score}, Baseline score: #{@prompt_diff.baseline_score}"
+        next msg
+      end
+
+      # Check dataset comparability — names, inputs, AND expected must match
+      unless @prompt_diff.cases_comparable?
+        unless @prompt_diff.case_names_match?
+          mm = @prompt_diff.mismatched_cases
+          msg += "  Case set mismatch — candidate and baseline must have identical cases:\n"
+          mm[:only_in_baseline].each { |n| msg += "    only in baseline: #{n}\n" }
+          mm[:only_in_candidate].each { |n| msg += "    only in candidate: #{n}\n" }
+        end
+        @prompt_diff.input_mismatches.each do |m|
+          msg += "  Input mismatch for '#{m[:case]}' — same name but different inputs\n"
+        end
+        @prompt_diff.expected_mismatches.each do |m|
+          msg += "  Expected mismatch for '#{m[:case]}' — same name/input but different expected values\n"
+        end
+        next msg
+      end
+
+      # Check per-case score regressions (even if global average is flat)
+      if @prompt_diff.score_regressions.any?
+        msg += "  Per-case score regressions (#{@prompt_diff.score_regressions.length}):\n"
+        @prompt_diff.score_regressions.each do |r|
+          msg += "    #{r[:case]}: #{r[:baseline_score]} -> #{r[:candidate_score]} (#{r[:delta]})\n"
+        end
+        msg += "  Score delta: #{@prompt_diff.score_delta}"
+        next msg
+      end
+
+      # Check pass/fail regressions and removed cases
+      removed = @prompt_diff.removed_passing_cases
+      reg_count = @prompt_diff.regressions.length + removed.length
+      msg += "  Found #{reg_count} regression(s):\n"
+      @prompt_diff.regressions.each do |r|
+        msg += "    #{r[:case]}: was PASS, now FAIL -- #{r[:detail]}\n"
+      end
+      removed.each do |name|
+        msg += "    #{name}: REMOVED (was passing in baseline)\n"
+      end
+      msg += "  Score delta: #{@prompt_diff.score_delta}"
+      next msg
+    end
+
     msg = format_failure_message(@eval_name, @error, @report, @minimum_score, @maximum_cost)
     if @diff&.regressed?
       msg += "\n\nRegressions from baseline:\n"
       @diff.regressions.each do |r|
-        msg += "  #{r[:case]}: was PASS, now FAIL — #{r[:detail]}\n"
+        msg += "  #{r[:case]}: was PASS, now FAIL -- #{r[:detail]}\n"
       end
       msg += "  Score delta: #{@diff.score_delta}"
     end
@@ -114,4 +185,14 @@ RSpec::Matchers.define :pass_eval do |eval_name|
   failure_message_when_negated do
     "expected #{@eval_name} eval NOT to pass, but it passed with score: #{@report.score.round(2)}"
   end
+
+  def sample_response_only_compare?
+    return false unless @comparison_step
+    return false if @context[:adapter] || @context[:model]
+
+    defn = @comparison_step.send(:all_eval_definitions)[@eval_name.to_s]
+    defn&.build_adapter
+  rescue StandardError
+    false
+  end
 end

data/lib/ruby_llm/contract/rspec.rb

@@ -13,11 +13,16 @@ RSpec.configure do |config|
   # Prevents non-block stub_all_steps from leaking between examples.
   config.around(:each) do |example|
     original_adapter = RubyLLM::Contract.configuration.default_adapter
+    original_logger = RubyLLM::Contract.configuration.logger
+    original_eval_hosts = RubyLLM::Contract.eval_hosts.dup
     original_overrides = RubyLLM::Contract.step_adapter_overrides.dup
     begin
       example.run
     ensure
       RubyLLM::Contract.configuration.default_adapter = original_adapter
+      RubyLLM::Contract.configuration.logger = original_logger
+      RubyLLM::Contract.reset_eval_hosts!
+      RubyLLM::Contract.eval_hosts.concat(original_eval_hosts)
       RubyLLM::Contract.step_adapter_overrides.replace(original_overrides)
     end
   end

data/lib/ruby_llm/contract/step/adapter_caller.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Step
+      class AdapterCaller
+        def initialize(adapter:, adapter_options:)
+          @adapter = adapter
+          @adapter_options = adapter_options
+        end
+
+        def call(messages)
+          start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          response = @adapter.call(messages: messages, **@adapter_options)
+          latency_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
+          [response, latency_ms]
+        rescue StandardError => error
+          [Result.new(status: :adapter_error, raw_output: nil, parsed_output: nil, validation_errors: [error.message]), 0]
+        end
+      end
+    end
+  end
+end