ruby_llm-contract 0.4.2 → 0.5.0

data/lib/ruby_llm/contract/eval/runner.rb

@@ -4,8 +4,6 @@ module RubyLLM
   module Contract
     module Eval
       class Runner
-        include TraitEvaluator
-        include ContractDetailBuilder
         include Concerns::ContextHelpers
 
         def self.run(step:, dataset:, context: {}, concurrency: nil)
@@ -20,17 +18,35 @@ module RubyLLM
         end
 
         def run
-          results = if @concurrency && @concurrency > 1
-                      run_concurrent
-                    else
-                      @dataset.cases.map { |test_case| evaluate_case(test_case) }
-                    end
-          step_name = @step.respond_to?(:name) ? @step.name : @step.to_s
-          Report.new(dataset_name: @dataset.name, results: results, step_name: step_name)
+          Report.new(
+            dataset_name: @dataset.name,
+            results: collected_results,
+            step_name: step_name
+          )
         end
 
         private
 
+        def collected_results
+          concurrent? ? run_concurrent : run_serial
+        end
+
+        def run_serial
+          @dataset.cases.map { |test_case| case_executor.call(test_case: test_case, context: @context) }
+        end
+
+        def concurrent?
+          @concurrency && @concurrency > 1
+        end
+
+        def step_name
+          @step.respond_to?(:name) ? @step.name : @step.to_s
+        end
+
+        def case_executor
+          @case_executor ||= CaseExecutor.new(step: @step)
+        end
+
         def run_concurrent
           require "concurrent"
           pool = Concurrent::FixedThreadPool.new(@concurrency)
@@ -39,10 +55,10 @@ module RubyLLM
           # gets a single-response adapter with its own response (by index).
           per_case_contexts = build_per_case_contexts
 
-          futures = @dataset.cases.each_with_index.map do |test_case, i|
-            ctx = per_case_contexts[i]
+          futures = @dataset.cases.each_with_index.map do |test_case, index|
+            case_context = per_case_contexts[index]
             Concurrent::Future.execute(executor: pool) do
-              evaluate_case_with_context(test_case, ctx)
+              case_executor.call(test_case: test_case, context: case_context)
             end
           end
           futures.map(&:value!)
@@ -55,10 +71,10 @@ module RubyLLM
           adapter = @context[:adapter]
           responses = adapter.respond_to?(:responses_array) ? adapter.responses_array : nil
 
-          @dataset.cases.each_with_index.map do |_, i|
+          @dataset.cases.each_with_index.map do |_, index|
             if responses
               # Give each case its own single-response adapter
-              response = responses[i] || responses.last
+              response = responses[index] || responses.last
               per_case_adapter = Adapters::Test.new(response: response)
               @context.merge(adapter: per_case_adapter)
             else
@@ -66,199 +82,6 @@ module RubyLLM
             end
           end
         end
-
-        def evaluate_case_with_context(test_case, context)
-          run_result = @step.run(test_case.input, context: context)
-          step_result = normalize_result(run_result)
-          eval_result = dispatch_evaluation(step_result, test_case)
-
-          result = build_case_result(test_case, step_result, eval_result)
-
-          if test_case.respond_to?(:step_expectations) && test_case.step_expectations &&
-             run_result.respond_to?(:outputs_by_step)
-            evaluate_step_expectations(result, run_result.outputs_by_step, test_case.step_expectations)
-          else
-            result
-          end
-        rescue RubyLLM::Contract::Error => e
-          raise unless e.message.include?("No adapter configured")
-
-          skipped_result(test_case, e.message)
-        end
-
-        def evaluate_case(test_case)
-          run_result = @step.run(test_case.input, context: @context)
-          step_result = normalize_result(run_result)
-          eval_result = dispatch_evaluation(step_result, test_case)
-
-          result = build_case_result(test_case, step_result, eval_result)
-
-          # Pipeline per-step evaluation
-          if test_case.respond_to?(:step_expectations) && test_case.step_expectations &&
-             run_result.respond_to?(:outputs_by_step)
-            evaluate_step_expectations(result, run_result.outputs_by_step, test_case.step_expectations)
-          else
-            result
-          end
-        rescue RubyLLM::Contract::Error => e
-          raise unless e.message.include?("No adapter configured")
-
-          skipped_result(test_case, e.message)
-        end
-
-        def build_case_result(test_case, step_result, eval_result)
-          trace = step_result.respond_to?(:trace) ? step_result.trace : nil
-          CaseResult.new(
-            name: test_case.name,
-            input: test_case.input,
-            output: step_result.parsed_output,
-            expected: test_case.expected,
-            step_status: step_result.status,
-            score: eval_result.score,
-            passed: eval_result.passed,
-            label: eval_result.label,
-            details: eval_result.details,
-            duration_ms: extract_latency(trace),
-            cost: extract_cost(trace)
-          )
-        end
-
-        def extract_latency(trace)
-          return nil unless trace
-
-          # Pipeline::Trace uses total_latency_ms, Step::Trace uses latency_ms
-          if trace.respond_to?(:total_latency_ms)
-            trace.total_latency_ms
-          else
-            trace[:latency_ms]
-          end
-        end
-
-        def extract_cost(trace)
-          return nil unless trace
-
-          # Pipeline::Trace uses total_cost, Step::Trace uses cost
-          if trace.respond_to?(:total_cost)
-            trace.total_cost
-          else
-            trace[:cost]
-          end
-        end
-
-        def dispatch_evaluation(step_result, test_case)
-          return contract_failure(step_result) unless step_result.ok?
-
-          if test_case.evaluator
-            evaluate_with_custom(step_result, test_case)
-          elsif test_case.expected_traits
-            evaluate_traits(step_result, test_case)
-          elsif !test_case.expected.nil?
-            evaluate_expected(step_result, test_case)
-          else
-            evaluate_contract_only
-          end
-        end
-
-        def normalize_result(result)
-          return result if result.respond_to?(:parsed_output)
-
-          normalize_pipeline_result(result)
-        end
-
-        def normalize_pipeline_result(result)
-          last_result = result.step_results&.last&.dig(:result)
-          is_ok = result.ok?
-          pipeline_trace = result.respond_to?(:trace) ? result.trace : nil
-
-          PipelineResultAdapter.new(
-            status: result.status,
-            ok_flag: is_ok,
-            parsed_output: is_ok ? result.outputs_by_step.values.last : nil,
-            validation_errors: last_result.respond_to?(:validation_errors) ? last_result.validation_errors : [],
-            trace: pipeline_trace || (last_result.respond_to?(:trace) ? last_result.trace : {})
-          )
-        end
-
-        def evaluate_expected(step_result, test_case)
-          dispatch_expected_evaluator(
-            output: step_result.parsed_output,
-            expected: test_case.expected,
-            input: test_case.input
-          )
-        end
-
-        def dispatch_expected_evaluator(output:, expected:, input:)
-          if expected.is_a?(Hash)
-            Evaluator::JsonIncludes.new.call(output: output, expected: expected, input: input)
-          elsif expected.is_a?(::Regexp)
-            Evaluator::Regex.new(expected).call(output: output, input: input)
-          else
-            Evaluator::Exact.new.call(output: output, expected: expected, input: input)
-          end
-        end
-
-        def evaluate_with_custom(step_result, test_case)
-          evaluator = test_case.evaluator
-          evaluator = Evaluator::ProcEvaluator.new(evaluator) if evaluator.is_a?(::Proc)
-          evaluator.call(output: step_result.parsed_output, expected: test_case.expected, input: test_case.input)
-        end
-
-        def evaluate_contract_only
-          EvaluationResult.new(score: 1.0, passed: true, details: build_contract_details)
-        end
-
-        def contract_failure(step_result)
-          EvaluationResult.new(
-            score: 0.0, passed: false,
-            details: "step failed: #{step_result.status} — #{step_result.validation_errors.join(", ")}"
-          )
-        end
-
-        def evaluate_step_expectations(result, outputs_by_step, expectations)
-          step_results = {}
-          all_passed = true
-
-          expectations.each do |step_alias, expected|
-            output = outputs_by_step[step_alias]
-            if output.nil?
-              step_results[step_alias] = { passed: false, details: "step not executed" }
-              all_passed = false
-            else
-              eval_res = dispatch_expected_evaluator(output: output, expected: expected, input: nil)
-              step_results[step_alias] = { passed: eval_res.passed, score: eval_res.score, details: eval_res.details }
-              all_passed = false unless eval_res.passed
-            end
-          end
-
-          # Rebuild CaseResult with step_results metadata
-          failed_steps = step_results.select { |_, v| !v[:passed] }
-          failure_details = failed_steps.map { |k, v| "#{k}: #{v[:details]}" }.join("; ")
-
-          CaseResult.new(
-            name: result.name, input: result.input, output: result.output,
-            expected: result.expected,
-            step_status: all_passed ? result.step_status : :step_expectation_failed,
-            score: all_passed ? result.score : 0.0,
-            passed: result.passed? && all_passed,
-            label: all_passed ? result.label : "FAIL",
-            details: all_passed ? result.details : "step expectations failed: #{failure_details}",
-            duration_ms: result.duration_ms, cost: result.cost
-          )
-        end
-
-        def skipped_result(test_case, reason)
-          CaseResult.new(
-            name: test_case.name,
-            input: test_case.input,
-            output: nil,
-            expected: test_case.expected,
-            step_status: :skipped,
-            score: 0.0,
-            passed: false,
-            label: "SKIP",
-            details: "skipped: #{reason}"
-          )
-        end
       end
     end
   end

data/lib/ruby_llm/contract/eval/step_expectation_applier.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      class StepExpectationApplier
+        def initialize(expectation_evaluator: ExpectationEvaluator.new)
+          @expectation_evaluator = expectation_evaluator
+        end
+
+        def call(result:, run_result:, test_case:)
+          return result unless applicable?(test_case, run_result)
+
+          expectation_results = evaluate_expectations(run_result.outputs_by_step, test_case.step_expectations)
+          return result if expectation_results.values.all? { |entry| entry[:passed] }
+
+          rebuild_result(result, failure_details_for(expectation_results))
+        end
+
+        private
+
+        def applicable?(test_case, run_result)
+          test_case.respond_to?(:step_expectations) &&
+            test_case.step_expectations &&
+            run_result.respond_to?(:outputs_by_step)
+        end
+
+        def evaluate_expectations(outputs_by_step, expectations)
+          expectations.each_with_object({}) do |(step_alias, expected), results|
+            output = outputs_by_step[step_alias]
+            results[step_alias] = evaluate_single_expectation(output, expected)
+          end
+        end
+
+        def evaluate_single_expectation(output, expected)
+          return { passed: false, details: "step not executed" } if output.nil?
+
+          evaluation = @expectation_evaluator.call(output: output, expected: expected, input: nil)
+          { passed: evaluation.passed, details: evaluation.details }
+        end
+
+        def failure_details_for(expectation_results)
+          expectation_results
+            .select { |_, entry| !entry[:passed] }
+            .map { |step_alias, entry| "#{step_alias}: #{entry[:details]}" }
+            .join("; ")
+        end
+
+        def rebuild_result(result, failure_details)
+          CaseResult.new(
+            name: result.name,
+            input: result.input,
+            output: result.output,
+            expected: result.expected,
+            step_status: :step_expectation_failed,
+            score: 0.0,
+            passed: false,
+            label: "FAIL",
+            details: "step expectations failed: #{failure_details}",
+            duration_ms: result.duration_ms,
+            cost: result.cost
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/step_result_normalizer.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      class StepResultNormalizer
+        def call(result)
+          return result if result.respond_to?(:parsed_output)
+
+          normalize_pipeline_result(result)
+        end
+
+        private
+
+        def normalize_pipeline_result(result)
+          last_result = result.step_results&.last&.dig(:result)
+          successful = result.ok?
+          trace = result.respond_to?(:trace) ? result.trace : nil
+
+          PipelineResultAdapter.new(
+            status: result.status,
+            ok_flag: successful,
+            parsed_output: successful ? result.outputs_by_step.values.last : nil,
+            validation_errors: validation_errors_for(last_result),
+            trace: trace || trace_for(last_result)
+          )
+        end
+
+        def validation_errors_for(result)
+          result.respond_to?(:validation_errors) ? result.validation_errors : []
+        end
+
+        def trace_for(result)
+          result.respond_to?(:trace) ? result.trace : {}
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval.rb

@@ -8,11 +8,24 @@ require_relative "eval/evaluator/json_includes"
 require_relative "eval/evaluator/proc_evaluator"
 require_relative "eval/dataset"
 require_relative "eval/pipeline_result_adapter"
+require_relative "eval/expectation_evaluator"
 require_relative "eval/trait_evaluator"
 require_relative "eval/contract_detail_builder"
+require_relative "eval/case_scorer"
+require_relative "eval/case_result_builder"
+require_relative "eval/step_result_normalizer"
+require_relative "eval/step_expectation_applier"
+require_relative "eval/case_executor"
 require_relative "eval/runner"
+require_relative "eval/report_stats"
+require_relative "eval/report_presenter"
+require_relative "eval/report_storage"
 require_relative "eval/report"
 require_relative "eval/eval_definition"
 require_relative "eval/model_comparison"
 require_relative "eval/baseline_diff"
+require_relative "eval/prompt_diff_serializer"
+require_relative "eval/prompt_diff_comparator"
+require_relative "eval/prompt_diff_presenter"
+require_relative "eval/prompt_diff"
 require_relative "eval/eval_history"

data/lib/ruby_llm/contract/minitest.rb

@@ -5,6 +5,20 @@ require "ruby_llm/contract"
 module RubyLLM
   module Contract
     module MinitestHelpers
+      # Snapshot adapter before each test so teardown can restore it.
+      def setup
+        super if defined?(super)
+        @_contract_original_adapter = RubyLLM::Contract.configuration.default_adapter
+      end
+
+      # Auto-cleanup: clear overrides AND restore original adapter.
+      # Prevents both non-block stub_step and stub_all_steps from leaking.
+      def teardown
+        RubyLLM::Contract.step_adapter_overrides.clear
+        RubyLLM::Contract.configuration.default_adapter = @_contract_original_adapter
+        super if defined?(super)
+      end
+
       def assert_satisfies_contract(result, msg = nil)
         assert result.ok?, msg || "Expected step result to satisfy contract, " \
           "but got status: #{result.status}. Errors: #{result.validation_errors.join(", ")}"
@@ -33,13 +47,113 @@ module RubyLLM
         report
       end
 
-      def stub_step(step_class, response: nil, responses: nil)
+      # Stub a specific step to return a canned response without API calls.
+      # Routes per-step — other steps are not affected.
+      #
+      #   stub_step(ClassifyTicket, response: { priority: "high" })
+      #
+      # Supports an optional block form — the override is removed after the
+      # block returns (even if it raises):
+      #
+      #   stub_step(ClassifyTicket, response: data) do
+      #     result = ClassifyTicket.run("test")
+      #   end
+      #   # ClassifyTicket.run no longer stubbed
+      #
+      def stub_step(step_class, response: nil, responses: nil, &block)
+        adapter = if responses
+                    Adapters::Test.new(responses: responses)
+                  else
+                    Adapters::Test.new(response: response)
+                  end
+
+        overrides = RubyLLM::Contract.step_adapter_overrides
+        previous = overrides[step_class]
+        overrides[step_class] = adapter
+
+        if block
+          begin
+            yield
+          ensure
+            if previous
+              overrides[step_class] = previous
+            else
+              overrides.delete(step_class)
+            end
+          end
+        end
+      end
+
+      # Stub multiple steps at once with different responses.
+      # Takes a hash of step_class => options. Requires a block.
+      #
+      #   stub_steps(
+      #     ClassifyTicket => { response: { priority: "high" } },
+      #     RouteToTeam => { response: { team: "billing" } }
+      #   ) do
+      #     result = TicketPipeline.run("test")
+      #   end
+      #
+      def stub_steps(stubs, &block)
+        raise ArgumentError, "stub_steps requires a block" unless block
+
+        overrides = RubyLLM::Contract.step_adapter_overrides
+        previous = {}
+
+        stubs.each do |step_class, opts|
+          opts = opts.transform_keys(&:to_sym)
+          adapter = if opts[:responses]
+                      Adapters::Test.new(responses: opts[:responses])
+                    else
+                      Adapters::Test.new(response: opts[:response])
+                    end
+          previous[step_class] = overrides[step_class]
+          overrides[step_class] = adapter
+        end
+
+        begin
+          yield
+        ensure
+          stubs.each_key do |step_class|
+            if previous[step_class]
+              overrides[step_class] = previous[step_class]
+            else
+              overrides.delete(step_class)
+            end
+          end
+        end
+      end
+
+      # Set a global test adapter for ALL steps.
+      #
+      #   stub_all_steps(response: { default: true })
+      #
+      # Supports an optional block form — the previous adapter is restored
+      # after the block returns (even if it raises):
+      #
+      #   stub_all_steps(response: { default: true }) do
+      #     # all steps use test adapter
+      #   end
+      #   # original adapter restored
+      #
+      def stub_all_steps(response: nil, responses: nil, &block)
         adapter = if responses
                     Adapters::Test.new(responses: responses)
                   else
                     Adapters::Test.new(response: response)
                   end
-        RubyLLM::Contract.configure { |c| c.default_adapter = adapter }
+
+        if block
+          previous = RubyLLM::Contract.configuration.default_adapter
+          begin
+            RubyLLM::Contract.configuration.default_adapter = adapter
+            yield
+          ensure
+            RubyLLM::Contract.configuration.default_adapter = previous
+          end
+        else
+          RubyLLM::Contract.configure { |c| c.default_adapter = adapter }
+        end
       end
     end
   end

data/lib/ruby_llm/contract/pipeline/base.rb

@@ -25,11 +25,24 @@ module RubyLLM
 
           # Internal mutable steps list for registration
           def steps_registry
-            @steps_registry ||= []
+            @steps_registry ||= begin
+              inherited_steps =
+                if superclass.respond_to?(:steps_registry, true)
+                  superclass.send(:steps_registry).map(&:dup)
+                else
+                  []
+                end
+
+              inherited_steps
+            end
           end
 
           def token_budget(limit = nil)
-            return @token_budget = limit if limit
+            if limit
+              raise ArgumentError, "token_budget must be positive, got #{limit}" unless limit.positive?
+
+              return @token_budget = limit
+            end
 
             @token_budget
           end

data/lib/ruby_llm/contract/rake_task.rb

@@ -7,7 +7,7 @@ module RubyLLM
   module Contract
     class RakeTask < ::Rake::TaskLib
       attr_accessor :name, :context, :fail_on_empty, :minimum_score, :maximum_cost,
-                    :eval_dirs, :save_baseline, :fail_on_regression
+                    :eval_dirs, :save_baseline, :fail_on_regression, :track_history
 
       def initialize(name = :"ruby_llm_contract:eval", &block)
         super()
@@ -19,6 +19,7 @@ module RubyLLM
         @eval_dirs = []      # directories to load eval files from (non-Rails)
         @save_baseline = false
         @fail_on_regression = false
+        @track_history = false
         block&.call(self)
         define_task
       end
@@ -47,18 +48,23 @@ module RubyLLM
           suite_cost = 0.0
 
           passed_reports = []
+          all_reports = []
 
           results.each do |host, reports|
             puts "\n#{host.name || host.to_s}"
             reports.each_value do |report|
               report.print_summary
               suite_cost += report.total_cost
+              all_reports << [host, report]
               report_ok = report_meets_score?(report) && !check_regression(report)
               gate_passed = false unless report_ok
               passed_reports << report if report_ok
             end
           end
 
+          # Save history BEFORE gating — failures are valuable trend data (ADR-0016 F3)
+          save_all_history!(all_reports, context) if @track_history
+
           if @maximum_cost && suite_cost > @maximum_cost
             abort "\nEval suite FAILED: total cost $#{format("%.4f", suite_cost)} " \
                   "exceeds budget $#{format("%.4f", @maximum_cost)}"
@@ -68,6 +74,7 @@ module RubyLLM
 
           # Save baselines only after ALL gates pass
           passed_reports.each { |r| save_baseline!(r) } if @save_baseline
+
           puts "\nAll evals passed."
         end
       end
@@ -98,6 +105,18 @@ module RubyLLM
         puts "  Baseline saved: #{path}"
       end
 
+      def save_all_history!(host_reports, context)
+        context_model = (context[:model] || context["model"]) if context.is_a?(Hash)
+        host_reports.each do |host, report|
+          # Model priority: context > step DSL > default config
+          model = context_model
+          model ||= (host.model if host.respond_to?(:model))
+          model ||= RubyLLM::Contract.configuration.default_model rescue nil
+          path = report.save_history!(model: model)
+          puts "  History saved: #{path}"
+        end
+      end
+
       def task_prerequisites
         defined?(::Rails) ? [:environment] : []
       end