ruby_llm-contract 0.2.3 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '080fd81afd87ad234cf66f7577080a4ac55a59f890e0c8c479479fccec57ad32'
-  data.tar.gz: cdabbac3ea1d81e1abd3cb850e927f410d98282bd23111be79463804ea4d84b9
+  metadata.gz: b032109a7818caa3f68cae651f9f99210765d4257825f52a332944a6120ad522
+  data.tar.gz: 8f4c1bb95cbcf79236723e100becf8c8f2b87061bd7c29827152e4d716a99ce3
 SHA512:
-  metadata.gz: 294b36f7264a2ba8b04334f3fd1c6b4433466a04c6be4aaccf23a92df3c7e92d04061ace018aa5243e28a9ef4fe64abc7f6de5ec11143c32bf5466bf591b9130
-  data.tar.gz: d7447319e3389264571209bc84d7dc84a441ffb76d1f64506d9cac2dc1953d26ba8cf3e1eb4169adf64576f1be7ae182bdf0c6e8e6b876220b102cea1e653fa6
+  metadata.gz: e84f8e58367e2eae1ea6a0a712e125be6b3edb361ce6feca984c659f15ca11ce658143adf7fdfcd09f5c1ff57d09fad31e431320f780dd08da7ab7499dd9b961
+  data.tar.gz: 29c98d8fb09a92df1a88136d7c67094784fdf2ae01ae9ec1aaa3fc5f1cd589fd27c7139c84663ba9e49c89e5537f98480eb451076c8a00dffcccfc3bf062f5d8

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## 0.3.0 (2026-03-23)
+
+Baseline regression detection — know when quality drops before users do.
+
+### Features
+
+- **`report.save_baseline!`** — serialize eval results to `.eval_baselines/` (JSON, git-tracked)
+- **`report.compare_with_baseline`** — returns `BaselineDiff` with regressions, improvements, score_delta, new/removed cases
+- **`diff.regressed?`** — true when any previously-passing case now fails
+- **`without_regressions` RSpec chain** — `expect(Step).to pass_eval("x").without_regressions`
+- **RakeTask `fail_on_regression`** — blocks CI when regressions detected
+- **RakeTask `save_baseline`** — auto-save after successful run
+- **Migration guide** — `docs/guide/migration.md` with 7 patterns for adopting the gem in existing Rails apps
+
+### Stats
+
+- 1086 tests, 0 failures
+
 ## 0.2.3 (2026-03-23)
 
 Production hardening from senior Rails review panel.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ruby_llm-contract (0.2.3)
+    ruby_llm-contract (0.3.0)
       dry-types (~> 1.7)
       ruby_llm (~> 1.0)
       ruby_llm-schema (~> 0.3)
@@ -165,7 +165,7 @@ CHECKSUMS
   rubocop-ast (1.49.1) sha256=4412f3ee70f6fe4546cc489548e0f6fcf76cafcfa80fa03af67098ffed755035
   ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33
   ruby_llm (1.14.0) sha256=57c6f7034fc4a44504ea137d70f853b07824f1c1cdbe774ab3ab3522e7098deb
-  ruby_llm-contract (0.2.3)
+  ruby_llm-contract (0.3.0)
   ruby_llm-schema (0.3.0) sha256=a591edc5ca1b7f0304f0e2261de61ba4b3bea17be09f5cf7558153adfda3dec6
   unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42
   unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f

data/README.md

@@ -111,6 +111,30 @@ end
 # bundle exec rake ruby_llm_contract:eval
 ```
 
+## Detect quality drops
+
+Save a baseline. Next run, see what regressed.
+
+```ruby
+report = ClassifyTicket.run_eval("regression", context: { model: "gpt-4.1-nano" })
+report.save_baseline!(model: "gpt-4.1-nano")
+
+# Later — after prompt change, model update, or provider weight shift:
+report = ClassifyTicket.run_eval("regression", context: { model: "gpt-4.1-nano" })
+diff = report.compare_with_baseline(model: "gpt-4.1-nano")
+
+diff.regressed?    # => true
+diff.regressions   # => [{case: "outage", baseline: {passed: true}, current: {passed: false}}]
+diff.score_delta   # => -0.33
+```
+
+```ruby
+# CI: block merge if any previously-passing case now fails
+expect(ClassifyTicket).to pass_eval("regression")
+  .with_context(model: "gpt-4.1-nano")
+  .without_regressions
+```
+
 ## Predict cost before running
 
 ```ruby
@@ -140,12 +164,13 @@ Works with any ruby_llm provider (OpenAI, Anthropic, Gemini, etc).
 | [Output Schema](docs/guide/output_schema.md) | Full schema reference + constraints |
 | [Pipeline](docs/guide/pipeline.md) | Multi-step composition, timeout, fail-fast |
 | [Testing](docs/guide/testing.md) | Test adapter, RSpec matchers |
+| [Migration](docs/guide/migration.md) | Adopting the gem in existing Rails apps |
 
 ## Roadmap
 
-**v0.2 (current):** Model comparison, cost tracking, eval with `add_case`, CI gating, Rails Railtie.
+**v0.3 (current):** Baseline regression detection — `save_baseline!`, `compare_with_baseline`, `without_regressions`. Migration guide.
 
-**v0.3:** Regression baselines — compare eval results with previous run, detect quality drift.
+**v0.2:** Model comparison, cost tracking, eval with `add_case`, CI gating, Rails Railtie.
 
 **v0.4:** Auto-routing — learn which model works for which input pattern.
 

data/lib/ruby_llm/contract/adapters/response.rb

@@ -4,11 +4,13 @@ module RubyLLM
   module Contract
     module Adapters
       class Response
+        include Concerns::DeepFreeze
+
         attr_reader :content, :usage
 
         def initialize(content:, usage: {})
-          @content = content
-          @usage = usage
+          @content = deep_dup_freeze(content)
+          @usage = deep_dup_freeze(usage)
           freeze
         end
       end

data/lib/ruby_llm/contract/adapters/test.rb

@@ -4,8 +4,9 @@ module RubyLLM
   module Contract
     module Adapters
       class Test < Base
-        def initialize(response: nil, responses: nil)
+        def initialize(response: nil, responses: nil, usage: nil)
           super()
+          @usage = (usage || { input_tokens: 0, output_tokens: 0 }).dup.freeze
           if responses
             raise ArgumentError, "responses: must not be empty (use response: nil for nil content)" if responses.empty?
 
@@ -36,7 +37,7 @@ module RubyLLM
                     else
                       @response
                     end
-          Response.new(content: content, usage: { input_tokens: 0, output_tokens: 0 })
+          Response.new(content: content, usage: @usage)
         end
       end
     end

data/lib/ruby_llm/contract/concerns/deep_freeze.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Concerns
+      # Deep-duplicate and freeze a value. Creates an independent frozen copy
+      # without mutating the original. Handles Hash, Array, String recursively.
+      module DeepFreeze
+        private
+
+        def deep_dup_freeze(obj)
+          case obj
+          when NilClass, Integer, Float, Symbol, TrueClass, FalseClass then obj
+          when Hash then obj.transform_values { |v| deep_dup_freeze(v) }.freeze
+          when Array then obj.map { |v| deep_dup_freeze(v) }.freeze
+          when String then obj.frozen? ? obj : obj.dup.freeze
+          else obj.frozen? ? obj : obj.dup.freeze
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/concerns/eval_host.rb

@@ -6,6 +6,7 @@ module RubyLLM
       module EvalHost
         def define_eval(name, &)
           @eval_definitions ||= {}
+          @file_sourced_evals ||= Set.new
           key = name.to_s
 
           if @eval_definitions.key?(key) && !Thread.current[:ruby_llm_contract_reloading]
@@ -14,12 +15,16 @@ module RubyLLM
           end
 
           @eval_definitions[key] = Eval::EvalDefinition.new(key, step_class: self, &)
+          @file_sourced_evals.add(key) if Thread.current[:ruby_llm_contract_reloading]
           Contract.register_eval_host(self)
           register_subclasses(self)
         end
 
-        def clear_eval_definitions!
-          @eval_definitions = {}
+        def clear_file_sourced_evals!
+          return unless defined?(@file_sourced_evals) && defined?(@eval_definitions)
+
+          @file_sourced_evals.each { |key| @eval_definitions.delete(key) }
+          @file_sourced_evals.clear
         end
 
         def eval_names
@@ -31,6 +36,7 @@ module RubyLLM
         end
 
         def run_eval(name = nil, context: {})
+          context ||= {}
           if name
             run_single_eval(name, context)
           else
@@ -39,6 +45,7 @@ module RubyLLM
         end
 
         def compare_models(eval_name, models:, context: {})
+          context ||= {}
           reports = models.each_with_object({}) do |model, hash|
             model_context = deep_dup_context(context).merge(model: model)
             hash[model] = run_single_eval(eval_name, model_context)
@@ -75,6 +82,7 @@ module RubyLLM
         end
 
         def eval_context(defn, context)
+          context = (context || {}).transform_keys { |k| k.respond_to?(:to_sym) ? k.to_sym : k }
           return context if context[:adapter]
 
           sample_adapter = defn.build_adapter

data/lib/ruby_llm/contract/eval/baseline_diff.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      class BaselineDiff
+        attr_reader :baseline_score, :current_score
+
+        def initialize(baseline_cases:, current_cases:)
+          @baseline = index_by_name(baseline_cases)
+          @current = index_by_name(current_cases)
+          @baseline_score = baseline_cases.empty? ? 0.0 : baseline_cases.sum { |c| c[:score] } / baseline_cases.length
+          @current_score = current_cases.empty? ? 0.0 : current_cases.sum { |c| c[:score] } / current_cases.length
+          freeze
+        end
+
+        def regressions
+          @baseline.filter_map do |name, baseline|
+            current = @current[name]
+            next unless current
+            next unless baseline[:passed] && !current[:passed]
+
+            {
+              case: name,
+              baseline: { passed: baseline[:passed], score: baseline[:score] },
+              current: { passed: current[:passed], score: current[:score] },
+              detail: current[:details]
+            }
+          end
+        end
+
+        def improvements
+          @baseline.filter_map do |name, baseline|
+            current = @current[name]
+            next unless current
+            next unless !baseline[:passed] && current[:passed]
+
+            {
+              case: name,
+              baseline: { passed: baseline[:passed], score: baseline[:score] },
+              current: { passed: current[:passed], score: current[:score] }
+            }
+          end
+        end
+
+        def score_delta
+          (current_score - baseline_score).round(4)
+        end
+
+        def regressed?
+          regressions.any?
+        end
+
+        def improved?
+          improvements.any?
+        end
+
+        def new_cases
+          (@current.keys - @baseline.keys)
+        end
+
+        def removed_cases
+          (@baseline.keys - @current.keys)
+        end
+
+        def to_s
+          lines = ["Score: #{baseline_score.round(2)} → #{current_score.round(2)} (#{format_delta})"]
+          regressions.each { |r| lines << "  REGRESSED  #{r[:case]}: #{r[:detail]}" }
+          improvements.each { |r| lines << "  IMPROVED   #{r[:case]}" }
+          new_cases.each { |c| lines << "  NEW        #{c}" }
+          removed_cases.each { |c| lines << "  REMOVED    #{c}" }
+          lines.join("\n")
+        end
+
+        private
+
+        def index_by_name(cases)
+          cases.each_with_object({}) { |c, h| h[c[:name]] = c }
+        end
+
+        def format_delta
+          d = score_delta
+          d >= 0 ? "+#{d}" : d.to_s
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/dataset.rb

@@ -23,8 +23,13 @@ module RubyLLM
         # dataset.case "name", input: {...}, expected_traits: {...}
         # dataset.case "name", input: {...}, evaluator: proc
         def add_case(name = nil, input:, expected: nil, expected_traits: nil, evaluator: nil)
+          case_name = name || "case_#{@cases.length + 1}"
+          if @cases.any? { |c| c.name == case_name }
+            raise ArgumentError, "Duplicate case name '#{case_name}'. Case names must be unique within a dataset."
+          end
+
           @cases << Case.new(
-            name: name || "case_#{@cases.length + 1}",
+            name: case_name,
             input: input,
             expected: expected,
             expected_traits: expected_traits,
@@ -37,13 +42,15 @@ module RubyLLM
       end
 
       class Case
+        include Concerns::DeepFreeze
+
         attr_reader :name, :input, :expected, :expected_traits, :evaluator
 
         def initialize(name:, input:, expected: nil, expected_traits: nil, evaluator: nil)
           @name = name
-          @input = input
-          @expected = expected
-          @expected_traits = expected_traits
+          @input = deep_dup_freeze(input)
+          @expected = deep_dup_freeze(expected)
+          @expected_traits = deep_dup_freeze(expected_traits)
           @evaluator = evaluator
           freeze
         end

data/lib/ruby_llm/contract/eval/eval_definition.rb

@@ -21,18 +21,19 @@ module RubyLLM
 
         def sample_response(response)
           @sample_response = response
+          @has_sample_response = true
           pre_validate_sample! if @step_class
         end
 
         def build_adapter
-          return nil unless @sample_response
+          return nil unless defined?(@has_sample_response) && @has_sample_response
 
-          Adapters::Test.new(response: @sample_response.is_a?(String) ? @sample_response : @sample_response.to_json)
+          Adapters::Test.new(response: @sample_response)
         end
 
         def add_case(description, input: nil, expected: nil, expected_traits: nil, evaluator: nil)
-          case_input = input || @default_input
-          raise ArgumentError, "add_case requires input (set default_input or pass input:)" unless case_input
+          case_input = input.nil? ? @default_input : input
+          raise ArgumentError, "add_case requires input (set default_input or pass input:)" if case_input.nil?
 
           @cases << {
             name: description,
@@ -44,12 +45,12 @@ module RubyLLM
         end
 
         def verify(description, expected_or_proc = nil, input: nil, expect: nil)
-          if expected_or_proc && expect
+          if !expected_or_proc.nil? && !expect.nil?
             raise ArgumentError, "verify accepts either a positional argument or expect: keyword, not both"
           end
 
-          expected_or_proc = expect if expect
-          case_input = input || @default_input
+          expected_or_proc = expect unless expect.nil?
+          case_input = input.nil? ? @default_input : input
           validate_verify_args!(expected_or_proc, case_input)
 
           evaluator = expected_or_proc.is_a?(::Proc) ? expected_or_proc : nil
@@ -78,15 +79,15 @@ module RubyLLM
 
         def effective_cases
           return @cases if @cases.any?
-          return [] unless @default_input
+          return [] if @default_input.nil?
 
           # Zero-verify: auto-add a contract check case
           [{ name: "contract check", input: @default_input, expected: nil, evaluator: nil }]
         end
 
         def validate_verify_args!(expected_or_proc, case_input)
-          raise ArgumentError, "verify requires either a positional argument or expect: keyword" unless expected_or_proc
-          raise ArgumentError, "verify requires input (set default_input or pass input:)" unless case_input
+          raise ArgumentError, "verify requires either a positional argument or expect: keyword" if expected_or_proc.nil?
+          raise ArgumentError, "verify requires input (set default_input or pass input:)" if case_input.nil?
         end
 
         def pre_validate_sample!

data/lib/ruby_llm/contract/eval/model_comparison.rb

@@ -8,7 +8,7 @@ module RubyLLM
 
         def initialize(eval_name:, reports:)
           @eval_name = eval_name
-          @reports = reports.freeze # { "model_name" => Report }
+          @reports = reports.dup.freeze # { "model_name" => Report }
           freeze
         end
 

data/lib/ruby_llm/contract/eval/report.rb

@@ -1,14 +1,18 @@
 # frozen_string_literal: true
 
+require "json"
+require "fileutils"
+
 module RubyLLM
   module Contract
     module Eval
       class Report
         attr_reader :dataset_name, :results
 
-        def initialize(dataset_name:, results:)
+        def initialize(dataset_name:, results:, step_name: nil)
           @dataset_name = dataset_name
-          @results = results.freeze
+          @step_name = step_name
+          @results = results.dup.freeze
           freeze
         end
 
@@ -78,6 +82,29 @@ module RubyLLM
           lines.join("\n")
         end
 
+        def save_baseline!(path: nil, model: nil)
+          file = path || default_baseline_path(model: model)
+          FileUtils.mkdir_p(File.dirname(file))
+          File.write(file, JSON.pretty_generate(serialize_for_baseline))
+          file
+        end
+
+        def compare_with_baseline(path: nil, model: nil)
+          file = path || default_baseline_path(model: model)
+          raise ArgumentError, "No baseline found at #{file}" unless File.exist?(file)
+
+          baseline_data = JSON.parse(File.read(file), symbolize_names: true)
+          validate_baseline!(baseline_data)
+          BaselineDiff.new(
+            baseline_cases: baseline_data[:cases],
+            current_cases: evaluated_results.map { |r| serialize_case(r) }
+          )
+        end
+
+        def baseline_exists?(path: nil, model: nil)
+          File.exist?(path || default_baseline_path(model: model))
+        end
+
         def print_summary(io = $stdout)
           io.puts summary
           io.puts
@@ -106,6 +133,48 @@ module RubyLLM
           results.reject { |r| r.step_status == :skipped }
         end
 
+        def default_baseline_path(model: nil)
+          parts = [".eval_baselines"]
+          parts << sanitize_name(@step_name) if @step_name
+          name = sanitize_name(dataset_name)
+          name = "#{name}_#{sanitize_name(model)}" if model
+          parts << "#{name}.json"
+          File.join(*parts)
+        end
+
+        def validate_baseline!(data)
+          if data[:dataset_name] && data[:dataset_name] != dataset_name
+            raise ArgumentError, "Baseline eval '#{data[:dataset_name]}' does not match '#{dataset_name}'"
+          end
+          if data[:step_name] && @step_name && data[:step_name] != @step_name
+            raise ArgumentError, "Baseline step '#{data[:step_name]}' does not match '#{@step_name}'"
+          end
+        end
+
+        def sanitize_name(name)
+          name.to_s.gsub(/[^a-zA-Z0-9_-]/, "_")
+        end
+
+        def serialize_for_baseline
+          {
+            dataset_name: dataset_name,
+            step_name: @step_name,
+            score: score,
+            total_cost: total_cost,
+            cases: evaluated_results.map { |r| serialize_case(r) }
+          }
+        end
+
+        def serialize_case(result)
+          {
+            name: result.name,
+            passed: result.passed?,
+            score: result.score,
+            details: result.details,
+            cost: result.cost
+          }
+        end
+
         def format_cost(cost)
           "$#{format("%.6f", cost)}"
         end

data/lib/ruby_llm/contract/eval/runner.rb

@@ -19,7 +19,8 @@ module RubyLLM
 
         def run
           results = @dataset.cases.map { |test_case| evaluate_case(test_case) }
-          Report.new(dataset_name: @dataset.name, results: results)
+          step_name = @step.respond_to?(:name) ? @step.name : @step.to_s
+          Report.new(dataset_name: @dataset.name, results: results, step_name: step_name)
         end
 
         private
@@ -81,7 +82,7 @@ module RubyLLM
             evaluate_with_custom(step_result, test_case)
           elsif test_case.expected_traits
             evaluate_traits(step_result, test_case)
-          elsif test_case.expected
+          elsif !test_case.expected.nil?
             evaluate_expected(step_result, test_case)
           else
             evaluate_contract_only

data/lib/ruby_llm/contract/eval.rb

@@ -14,3 +14,4 @@ require_relative "eval/runner"
 require_relative "eval/report"
 require_relative "eval/eval_definition"
 require_relative "eval/model_comparison"
+require_relative "eval/baseline_diff"

data/lib/ruby_llm/contract/pipeline/base.rb

@@ -20,7 +20,7 @@ module RubyLLM
           end
 
           def steps
-            steps_registry.dup.freeze
+            steps_registry.map { |s| s.dup.freeze }.freeze
           end
 
           # Internal mutable steps list for registration

data/lib/ruby_llm/contract/pipeline/runner.rb

@@ -13,7 +13,7 @@ module RubyLLM
           raise ArgumentError, "Pipeline has no steps defined" if steps.empty?
 
           @steps = steps
-          @context = context
+          @context = context || {}
           @timeout_ms = timeout_ms
           @token_budget = token_budget
         end

data/lib/ruby_llm/contract/pipeline/trace.rb

@@ -5,14 +5,15 @@ module RubyLLM
     module Pipeline
       class Trace
         include Concerns::TraceEquality
+        include Concerns::DeepFreeze
 
         attr_reader :trace_id, :total_latency_ms, :total_usage, :step_traces, :total_cost
 
         def initialize(trace_id: nil, total_latency_ms: nil, total_usage: nil, step_traces: nil)
           @trace_id = trace_id
           @total_latency_ms = total_latency_ms
-          @total_usage = total_usage
-          @step_traces = step_traces
+          @total_usage = deep_dup_freeze(total_usage)
+          @step_traces = step_traces&.dup&.freeze
           @total_cost = calculate_total_cost
           freeze
         end

data/lib/ruby_llm/contract/prompt/node.rb

@@ -7,8 +7,8 @@ module RubyLLM
         attr_reader :type, :content
 
         def initialize(type:, content:)
-          @type = type.freeze
-          @content = content.freeze
+          @type = type
+          @content = content.frozen? ? content : content.dup.freeze
           freeze
         end
 

data/lib/ruby_llm/contract/prompt/nodes/example_node.rb

@@ -8,8 +8,8 @@ module RubyLLM
           attr_reader :input, :output
 
           def initialize(input:, output:)
-            @input = input.freeze
-            @output = output.freeze
+            @input = input.frozen? ? input : input.dup.freeze
+            @output = output.frozen? ? output : output.dup.freeze
             super(type: :example, content: nil)
           end
 

data/lib/ruby_llm/contract/rake_task.rb

@@ -6,7 +6,8 @@ require "rake/tasklib"
 module RubyLLM
   module Contract
     class RakeTask < ::Rake::TaskLib
-      attr_accessor :name, :context, :fail_on_empty, :minimum_score, :maximum_cost, :eval_dirs
+      attr_accessor :name, :context, :fail_on_empty, :minimum_score, :maximum_cost,
+                    :eval_dirs, :save_baseline, :fail_on_regression
 
       def initialize(name = :"ruby_llm_contract:eval", &block)
         super()
@@ -16,6 +17,8 @@ module RubyLLM
         @minimum_score = nil # nil = require 100%; float = threshold
         @maximum_cost = nil  # nil = no cost limit; float = budget cap (suite-level)
         @eval_dirs = []      # directories to load eval files from (non-Rails)
+        @save_baseline = false
+        @fail_on_regression = false
         block&.call(self)
         define_task
       end
@@ -26,8 +29,7 @@ module RubyLLM
         desc "Run all ruby_llm-contract evals"
         task(@name => task_prerequisites) do
           require "ruby_llm/contract"
-          @eval_dirs.each { |dir| RubyLLM::Contract.load_evals!(dir) }
-          RubyLLM::Contract.load_evals!
+          RubyLLM::Contract.load_evals!(*@eval_dirs)
 
           results = RubyLLM::Contract.run_all_evals(context: @context)
 
@@ -43,12 +45,16 @@ module RubyLLM
           gate_passed = true
           suite_cost = 0.0
 
+          passed_reports = []
+
           results.each do |host, reports|
             puts "\n#{host.name || host.to_s}"
             reports.each_value do |report|
               report.print_summary
               suite_cost += report.total_cost
-              gate_passed = false unless report_meets_score?(report)
+              report_ok = report_meets_score?(report) && !check_regression(report)
+              gate_passed = false unless report_ok
+              passed_reports << report if report_ok
             end
           end
 
@@ -58,6 +64,9 @@ module RubyLLM
           end
 
           abort "\nEval suite FAILED" unless gate_passed
+
+          # Save baselines only after ALL gates pass
+          passed_reports.each { |r| save_baseline!(r) } if @save_baseline
           puts "\nAll evals passed."
         end
       end
@@ -70,6 +79,24 @@ module RubyLLM
         end
       end
 
+      def check_regression(report)
+        return false unless @fail_on_regression && report.baseline_exists?
+
+        diff = report.compare_with_baseline
+        if diff.regressed?
+          puts "\n  REGRESSIONS DETECTED:"
+          puts "  #{diff}"
+          true
+        else
+          false
+        end
+      end
+
+      def save_baseline!(report)
+        path = report.save_baseline!
+        puts "  Baseline saved: #{path}"
+      end
+
       def task_prerequisites
         Rake::Task.task_defined?(:environment) ? [:environment] : []
       end

data/lib/ruby_llm/contract/rspec/helpers.rb

@@ -10,18 +10,38 @@ module RubyLLM
         #   result = ClassifyTicket.run("test")
         #   result.parsed_output  # => {priority: "high"}
         #
-        # For multiple sequential responses:
-        #   stub_step(ClassifyTicket, responses: [{ a: 1 }, { a: 2 }])
+        # Only affects the specified step — other steps are not affected.
         #
         def stub_step(step_class, response: nil, responses: nil)
-          adapter = if responses
-                      Adapters::Test.new(responses: responses.map { |r| r.is_a?(String) ? r : r.to_json })
-                    else
-                      content = response.is_a?(String) ? response : response.to_json
-                      Adapters::Test.new(response: content)
-                    end
+          adapter = build_test_adapter(response: response, responses: responses)
+          allow(step_class).to receive(:run).and_wrap_original do |original, input, **kwargs|
+            context = (kwargs[:context] || {}).merge(adapter: adapter)
+            original.call(input, context: context)
+          end
+        end
+
+        # Set a global test adapter for ALL steps.
+        #
+        #   stub_all_steps(response: { default: true })
+        #
+        def stub_all_steps(response: nil, responses: nil)
+          adapter = build_test_adapter(response: response, responses: responses)
           RubyLLM::Contract.configure { |c| c.default_adapter = adapter }
         end
+
+        private
+
+        def build_test_adapter(response: nil, responses: nil)
+          if responses
+            Adapters::Test.new(responses: responses.map { |r| normalize_test_response(r) })
+          else
+            Adapters::Test.new(response: normalize_test_response(response))
+          end
+        end
+
+        def normalize_test_response(value)
+          value
+        end
       end
     end
   end

data/lib/ruby_llm/contract/rspec/pass_eval.rb

@@ -64,12 +64,18 @@ RSpec::Matchers.define :pass_eval do |eval_name|
     @maximum_cost = cost
   end
 
+  chain :without_regressions do
+    @check_regressions = true
+  end
+
   match do |step_or_pipeline|
     @eval_name = eval_name
     @context ||= {}
     @minimum_score ||= nil
     @maximum_cost ||= nil
+    @check_regressions ||= false
     @error = nil
+    @diff = nil
     @report = step_or_pipeline.run_eval(eval_name, context: @context)
 
     score_ok = if @minimum_score
@@ -80,14 +86,29 @@ RSpec::Matchers.define :pass_eval do |eval_name|
 
     cost_ok = @maximum_cost ? @report.total_cost <= @maximum_cost : true
 
-    score_ok && cost_ok
+    regression_ok = if @check_regressions && @report.baseline_exists?
+                      @diff = @report.compare_with_baseline
+                      !@diff.regressed?
+                    else
+                      true
+                    end
+
+    score_ok && cost_ok && regression_ok
   rescue StandardError => e
     @error = e
     false
   end
 
   failure_message do
-    format_failure_message(@eval_name, @error, @report, @minimum_score, @maximum_cost)
+    msg = format_failure_message(@eval_name, @error, @report, @minimum_score, @maximum_cost)
+    if @diff&.regressed?
+      msg += "\n\nRegressions from baseline:\n"
+      @diff.regressions.each do |r|
+        msg += "  #{r[:case]}: was PASS, now FAIL — #{r[:detail]}\n"
+      end
+      msg += "  Score delta: #{@diff.score_delta}"
+    end
+    msg
   end
 
   failure_message_when_negated do

data/lib/ruby_llm/contract/step/base.rb

@@ -58,18 +58,23 @@ module RubyLLM
             end
           end
 
-          KNOWN_CONTEXT_KEYS = %i[adapter model temperature max_tokens schema provider assume_model_exists].freeze
+          KNOWN_CONTEXT_KEYS = %i[adapter model temperature provider assume_model_exists].freeze
 
           def run(input, context: {})
+            context = (context || {}).transform_keys { |k| k.respond_to?(:to_sym) ? k.to_sym : k }
             warn_unknown_context_keys(context)
             adapter = resolve_adapter(context)
             default_model = context[:model] || model || RubyLLM::Contract.configuration.default_model
             policy = retry_policy
 
+            ctx_temp = context[:temperature]
+            extra = context.slice(:provider, :assume_model_exists)
             result = if policy
-                       run_with_retry(input, adapter: adapter, default_model: default_model, policy: policy)
+                       run_with_retry(input, adapter: adapter, default_model: default_model,
+                                      policy: policy, context_temperature: ctx_temp, extra_options: extra)
                      else
-                       run_once(input, adapter: adapter, model: default_model, context_temperature: context[:temperature])
+                       run_once(input, adapter: adapter, model: default_model,
+                                context_temperature: ctx_temp, extra_options: extra)
                      end
 
             invoke_around_call(input, result)
@@ -101,14 +106,14 @@ module RubyLLM
                                             "{ |c| c.default_adapter = ... } or pass context: { adapter: ... }"
           end
 
-          def run_once(input, adapter:, model:, context_temperature: nil)
+          def run_once(input, adapter:, model:, context_temperature: nil, extra_options: {})
             effective_temp = context_temperature || temperature
             Runner.new(
               input_type: input_type, output_type: output_type,
               prompt_block: prompt, contract_definition: effective_contract,
               adapter: adapter, model: model, output_schema: output_schema,
               max_output: max_output, max_input: max_input, max_cost: max_cost,
-              temperature: effective_temp
+              temperature: effective_temp, extra_options: extra_options
             ).call(input)
           rescue ArgumentError => e
             Result.new(status: :input_error, raw_output: nil, parsed_output: nil,

data/lib/ruby_llm/contract/step/dsl.rb

@@ -168,7 +168,7 @@ module RubyLLM
         end
 
         def retry_policy(models: nil, attempts: nil, retry_on: nil, &block)
-          if block || models || attempts
+          if block || models || attempts || retry_on
             return @retry_policy = RetryPolicy.new(models: models, attempts: attempts, retry_on: retry_on, &block)
           end
 

data/lib/ruby_llm/contract/step/retry_executor.rb

@@ -8,12 +8,13 @@ module RubyLLM
       module RetryExecutor
         private
 
-        def run_with_retry(input, adapter:, default_model:, policy:)
+        def run_with_retry(input, adapter:, default_model:, policy:, context_temperature: nil, extra_options: {})
           all_attempts = []
 
           policy.max_attempts.times do |attempt_index|
             model = policy.model_for_attempt(attempt_index, default_model)
-            result = run_once(input, adapter: adapter, model: model)
+            result = run_once(input, adapter: adapter, model: model,
+                              context_temperature: context_temperature, extra_options: extra_options)
             all_attempts << { attempt: attempt_index + 1, model: model, result: result }
             break unless policy.retryable?(result)
           end

data/lib/ruby_llm/contract/step/retry_policy.rb

@@ -15,6 +15,7 @@ module RubyLLM
           if block
             @max_attempts = 1
             instance_eval(&block)
+            warn_no_retry! if @max_attempts == 1 && @models.empty?
           else
             apply_keywords(models: models, attempts: attempts, retry_on: retry_on)
           end
@@ -65,6 +66,11 @@ module RubyLLM
           @retryable_statuses = Array(retry_on).dup if retry_on
         end
 
+        def warn_no_retry!
+          warn "[ruby_llm-contract] retry_policy has max_attempts=1 with no models. " \
+               "This means no actual retry will happen. Add `attempts 2` or `escalate %w[model1 model2]`."
+        end
+
         def validate_max_attempts!
           return if @max_attempts.is_a?(Integer) && @max_attempts >= 1
 

data/lib/ruby_llm/contract/step/runner.rb

@@ -8,7 +8,7 @@ module RubyLLM
 
         def initialize(input_type:, output_type:, prompt_block:, contract_definition:,
                        adapter:, model:, output_schema: nil, max_output: nil,
-                       max_input: nil, max_cost: nil, temperature: nil)
+                       max_input: nil, max_cost: nil, temperature: nil, extra_options: {})
           @input_type = input_type
           @output_type = output_type
           @prompt_block = prompt_block
@@ -20,6 +20,7 @@ module RubyLLM
           @max_input = max_input
           @max_cost = max_cost
           @temperature = temperature
+          @extra_options = extra_options
         end
 
         def call(input)
@@ -86,6 +87,7 @@ module RubyLLM
             opts[:schema] = @output_schema if @output_schema
             opts[:max_tokens] = @max_output if @max_output
             opts[:temperature] = @temperature if @temperature
+            @extra_options.each { |k, v| opts[k] = v unless opts.key?(k) }
           end
         end
 

data/lib/ruby_llm/contract/step/trace.rb

@@ -5,15 +5,16 @@ module RubyLLM
     module Step
       class Trace
         include Concerns::TraceEquality
+        include Concerns::DeepFreeze
 
         attr_reader :messages, :model, :latency_ms, :usage, :attempts, :cost
 
         def initialize(messages: nil, model: nil, latency_ms: nil, usage: nil, attempts: nil, cost: nil)
-          @messages = messages
-          @model = model
+          @messages = deep_dup_freeze(messages)
+          @model = model.frozen? ? model : model&.dup&.freeze
           @latency_ms = latency_ms
-          @usage = usage
-          @attempts = attempts
+          @usage = deep_dup_freeze(usage)
+          @attempts = deep_dup_freeze(attempts)
           @cost = cost || CostCalculator.calculate(model_name: model, usage: usage)
           freeze
         end

data/lib/ruby_llm/contract/version.rb

@@ -2,6 +2,6 @@
 
 module RubyLLM
   module Contract
-    VERSION = "0.2.3"
+    VERSION = "0.3.0"
   end
 end

data/lib/ruby_llm/contract.rb

@@ -40,25 +40,21 @@ module RubyLLM
         @eval_hosts = []
       end
 
-      def load_evals!(dir = nil)
-        dirs = if dir
-                 [dir]
-               elsif defined?(::Rails)
-                 %w[app/steps/eval app/contracts/eval].filter_map do |path|
-                   full = ::Rails.root.join(path)
-                   full.to_s if full.exist?
-                 end
-               else
-                 []
-               end
+      def load_evals!(*dirs)
+        dirs = dirs.flatten.compact
+        if dirs.empty? && defined?(::Rails)
+          dirs = %w[app/steps/eval app/contracts/eval].filter_map do |path|
+            full = ::Rails.root.join(path)
+            full.to_s if full.exist?
+          end
+        end
 
         return if dirs.empty?
 
-        # Clear existing eval definitions before reload to prevent stale state.
-        # Thread-local flag suppresses the "redefining" warning during reload.
+        # Clear file-sourced evals ONCE, then load ALL dirs.
         Thread.current[:ruby_llm_contract_reloading] = true
         eval_hosts.each do |host|
-          host.clear_eval_definitions! if host.respond_to?(:clear_eval_definitions!)
+          host.clear_file_sourced_evals! if host.respond_to?(:clear_file_sourced_evals!)
         end
 
         dirs.each do |d|
@@ -70,11 +66,17 @@ module RubyLLM
 
       private
 
-      # Filter out GC'd anonymous classes and classes that no longer have evals
+      # Filter stale hosts, deduplicate by name (last wins), prune registry in-place
       def live_eval_hosts
-        eval_hosts.select do |host|
-          host.respond_to?(:eval_defined?) && host.eval_defined?
-        end
+        # Remove hosts without evals
+        @eval_hosts&.reject! { |h| !h.respond_to?(:eval_defined?) || !h.eval_defined? }
+
+        # Deduplicate: if two classes share a name (reload), keep the latest
+        seen = {}
+        @eval_hosts&.each { |h| seen[h.name || h.object_id] = h }
+        @eval_hosts = seen.values
+
+        @eval_hosts || []
       end
 
       def auto_create_adapter!
@@ -87,6 +89,7 @@ module RubyLLM
   end
 end
 
+require_relative "contract/concerns/deep_freeze"
 require_relative "contract/concerns/deep_symbolize"
 require_relative "contract/concerns/eval_host"
 require_relative "contract/concerns/trace_equality"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby_llm-contract
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.0
 platform: ruby
 authors:
 - Justyna
@@ -82,6 +82,7 @@ files:
 - lib/ruby_llm/contract/adapters/response.rb
 - lib/ruby_llm/contract/adapters/ruby_llm.rb
 - lib/ruby_llm/contract/adapters/test.rb
+- lib/ruby_llm/contract/concerns/deep_freeze.rb
 - lib/ruby_llm/contract/concerns/deep_symbolize.rb
 - lib/ruby_llm/contract/concerns/eval_host.rb
 - lib/ruby_llm/contract/concerns/trace_equality.rb
@@ -97,6 +98,7 @@ files:
 - lib/ruby_llm/contract/dsl.rb
 - lib/ruby_llm/contract/errors.rb
 - lib/ruby_llm/contract/eval.rb
+- lib/ruby_llm/contract/eval/baseline_diff.rb
 - lib/ruby_llm/contract/eval/case_result.rb
 - lib/ruby_llm/contract/eval/contract_detail_builder.rb
 - lib/ruby_llm/contract/eval/dataset.rb