ruby_llm-contract 0.2.3 → 0.3.6

data/lib/ruby_llm/contract/eval/report.rb

@@ -1,14 +1,18 @@
 # frozen_string_literal: true
 
+require "json"
+require "fileutils"
+
 module RubyLLM
   module Contract
     module Eval
       class Report
         attr_reader :dataset_name, :results
 
-        def initialize(dataset_name:, results:)
+        def initialize(dataset_name:, results:, step_name: nil)
           @dataset_name = dataset_name
-          @results = results.freeze
+          @step_name = step_name
+          @results = results.dup.freeze
           freeze
         end
 
@@ -78,6 +82,29 @@ module RubyLLM
           lines.join("\n")
         end
 
+        def save_baseline!(path: nil, model: nil)
+          file = path || default_baseline_path(model: model)
+          FileUtils.mkdir_p(File.dirname(file))
+          File.write(file, JSON.pretty_generate(serialize_for_baseline))
+          file
+        end
+
+        def compare_with_baseline(path: nil, model: nil)
+          file = path || default_baseline_path(model: model)
+          raise ArgumentError, "No baseline found at #{file}" unless File.exist?(file)
+
+          baseline_data = JSON.parse(File.read(file), symbolize_names: true)
+          validate_baseline!(baseline_data)
+          BaselineDiff.new(
+            baseline_cases: baseline_data[:cases],
+            current_cases: results.map { |r| serialize_case(r) }
+          )
+        end
+
+        def baseline_exists?(path: nil, model: nil)
+          File.exist?(path || default_baseline_path(model: model))
+        end
+
         def print_summary(io = $stdout)
           io.puts summary
           io.puts
@@ -106,6 +133,48 @@ module RubyLLM
           results.reject { |r| r.step_status == :skipped }
         end
 
+        def default_baseline_path(model: nil)
+          parts = [".eval_baselines"]
+          parts << sanitize_name(@step_name) if @step_name
+          name = sanitize_name(dataset_name)
+          name = "#{name}_#{sanitize_name(model)}" if model
+          parts << "#{name}.json"
+          File.join(*parts)
+        end
+
+        def validate_baseline!(data)
+          if data[:dataset_name] && data[:dataset_name] != dataset_name
+            raise ArgumentError, "Baseline eval '#{data[:dataset_name]}' does not match '#{dataset_name}'"
+          end
+          if data[:step_name] && @step_name && data[:step_name] != @step_name
+            raise ArgumentError, "Baseline step '#{data[:step_name]}' does not match '#{@step_name}'"
+          end
+        end
+
+        def sanitize_name(name)
+          name.to_s.gsub(/[^a-zA-Z0-9_-]/, "_")
+        end
+
+        def serialize_for_baseline
+          {
+            dataset_name: dataset_name,
+            step_name: @step_name,
+            score: score,
+            total_cost: total_cost,
+            cases: evaluated_results.map { |r| serialize_case(r) }
+          }
+        end
+
+        def serialize_case(result)
+          {
+            name: result.name,
+            passed: result.passed?,
+            score: result.score,
+            details: result.details,
+            cost: result.cost
+          }
+        end
+
         def format_cost(cost)
           "$#{format("%.6f", cost)}"
         end

data/lib/ruby_llm/contract/eval/runner.rb

@@ -19,7 +19,8 @@ module RubyLLM
 
         def run
           results = @dataset.cases.map { |test_case| evaluate_case(test_case) }
-          Report.new(dataset_name: @dataset.name, results: results)
+          step_name = @step.respond_to?(:name) ? @step.name : @step.to_s
+          Report.new(dataset_name: @dataset.name, results: results, step_name: step_name)
         end
 
         private
@@ -31,7 +32,8 @@ module RubyLLM
 
           build_case_result(test_case, step_result, eval_result)
         rescue RubyLLM::Contract::Error => e
-          # No adapter configured — skip this case (offline mode without sample_response)
+          raise unless e.message.include?("No adapter configured")
+
           skipped_result(test_case, e.message)
         end
 
@@ -81,7 +83,7 @@ module RubyLLM
             evaluate_with_custom(step_result, test_case)
           elsif test_case.expected_traits
             evaluate_traits(step_result, test_case)
-          elsif test_case.expected
+          elsif !test_case.expected.nil?
             evaluate_expected(step_result, test_case)
           else
             evaluate_contract_only

data/lib/ruby_llm/contract/eval/trait_evaluator.rb

@@ -26,6 +26,8 @@ module RubyLLM
 
         def trait_error(key, value, expectation)
           case expectation
+          when ::Proc
+            trait_proc_error(key, value, expectation)
           when ::Regexp
             trait_regexp_error(key, value, expectation)
           when Range
@@ -56,6 +58,10 @@ module RubyLLM
           "#{key}: expected falsy, got #{value.inspect}" if value
         end
 
+        def trait_proc_error(key, value, expectation)
+          "#{key}: trait check failed, got #{value.inspect}" unless expectation.call(value)
+        end
+
         def trait_equality_error(key, value, expectation)
           "#{key}: expected #{expectation.inspect}, got #{value.inspect}" unless value == expectation
         end

data/lib/ruby_llm/contract/eval.rb

@@ -14,3 +14,4 @@ require_relative "eval/runner"
 require_relative "eval/report"
 require_relative "eval/eval_definition"
 require_relative "eval/model_comparison"
+require_relative "eval/baseline_diff"

data/lib/ruby_llm/contract/pipeline/base.rb

@@ -20,7 +20,7 @@ module RubyLLM
           end
 
           def steps
-            steps_registry.dup.freeze
+            steps_registry.map { |s| s.dup.freeze }.freeze
           end
 
           # Internal mutable steps list for registration

data/lib/ruby_llm/contract/pipeline/result.rb

@@ -116,7 +116,7 @@ module RubyLLM
         end
 
         def format_output(output)
-          return ["(no output)"] unless output
+          return ["(no output)"] if output.nil?
 
           pairs = output.is_a?(Hash) ? output : { value: output }
           pairs.map do |key, val|

data/lib/ruby_llm/contract/pipeline/runner.rb

@@ -13,7 +13,7 @@ module RubyLLM
           raise ArgumentError, "Pipeline has no steps defined" if steps.empty?
 
           @steps = steps
-          @context = context
+          @context = context || {}
           @timeout_ms = timeout_ms
           @token_budget = token_budget
         end

data/lib/ruby_llm/contract/pipeline/trace.rb

@@ -5,14 +5,15 @@ module RubyLLM
     module Pipeline
       class Trace
         include Concerns::TraceEquality
+        include Concerns::DeepFreeze
 
         attr_reader :trace_id, :total_latency_ms, :total_usage, :step_traces, :total_cost
 
         def initialize(trace_id: nil, total_latency_ms: nil, total_usage: nil, step_traces: nil)
           @trace_id = trace_id
           @total_latency_ms = total_latency_ms
-          @total_usage = total_usage
-          @step_traces = step_traces
+          @total_usage = deep_dup_freeze(total_usage)
+          @step_traces = step_traces&.dup&.freeze
           @total_cost = calculate_total_cost
           freeze
         end

data/lib/ruby_llm/contract/prompt/builder.rb

@@ -10,7 +10,8 @@ module RubyLLM
         end
 
         def build(input = nil)
-          if input && @block.arity >= 1
+          @nodes = []
+          if !input.nil? && @block.arity >= 1
             instance_exec(input, &@block)
           else
             instance_eval(&@block)

data/lib/ruby_llm/contract/prompt/node.rb

@@ -7,8 +7,8 @@ module RubyLLM
         attr_reader :type, :content
 
         def initialize(type:, content:)
-          @type = type.freeze
-          @content = content.freeze
+          @type = type
+          @content = content.frozen? ? content : content.dup.freeze
           freeze
         end
 

data/lib/ruby_llm/contract/prompt/nodes/example_node.rb

@@ -8,8 +8,8 @@ module RubyLLM
           attr_reader :input, :output
 
           def initialize(input:, output:)
-            @input = input.freeze
-            @output = output.freeze
+            @input = input.frozen? ? input : input.dup.freeze
+            @output = output.frozen? ? output : output.dup.freeze
             super(type: :example, content: nil)
           end
 

data/lib/ruby_llm/contract/rake_task.rb

@@ -6,7 +6,8 @@ require "rake/tasklib"
 module RubyLLM
   module Contract
     class RakeTask < ::Rake::TaskLib
-      attr_accessor :name, :context, :fail_on_empty, :minimum_score, :maximum_cost, :eval_dirs
+      attr_accessor :name, :context, :fail_on_empty, :minimum_score, :maximum_cost,
+                    :eval_dirs, :save_baseline, :fail_on_regression
 
       def initialize(name = :"ruby_llm_contract:eval", &block)
         super()
@@ -16,6 +17,8 @@ module RubyLLM
         @minimum_score = nil # nil = require 100%; float = threshold
         @maximum_cost = nil  # nil = no cost limit; float = budget cap (suite-level)
         @eval_dirs = []      # directories to load eval files from (non-Rails)
+        @save_baseline = false
+        @fail_on_regression = false
         block&.call(self)
         define_task
       end
@@ -26,8 +29,7 @@ module RubyLLM
         desc "Run all ruby_llm-contract evals"
         task(@name => task_prerequisites) do
           require "ruby_llm/contract"
-          @eval_dirs.each { |dir| RubyLLM::Contract.load_evals!(dir) }
-          RubyLLM::Contract.load_evals!
+          RubyLLM::Contract.load_evals!(*@eval_dirs)
 
           results = RubyLLM::Contract.run_all_evals(context: @context)
 
@@ -43,12 +45,16 @@ module RubyLLM
           gate_passed = true
           suite_cost = 0.0
 
+          passed_reports = []
+
           results.each do |host, reports|
             puts "\n#{host.name || host.to_s}"
             reports.each_value do |report|
               report.print_summary
               suite_cost += report.total_cost
-              gate_passed = false unless report_meets_score?(report)
+              report_ok = report_meets_score?(report) && !check_regression(report)
+              gate_passed = false unless report_ok
+              passed_reports << report if report_ok
             end
           end
 
@@ -58,6 +64,9 @@ module RubyLLM
           end
 
           abort "\nEval suite FAILED" unless gate_passed
+
+          # Save baselines only after ALL gates pass
+          passed_reports.each { |r| save_baseline!(r) } if @save_baseline
           puts "\nAll evals passed."
         end
       end
@@ -70,6 +79,24 @@ module RubyLLM
         end
       end
 
+      def check_regression(report)
+        return false unless @fail_on_regression && report.baseline_exists?
+
+        diff = report.compare_with_baseline
+        if diff.regressed?
+          puts "\n  REGRESSIONS DETECTED:"
+          puts "  #{diff}"
+          true
+        else
+          false
+        end
+      end
+
+      def save_baseline!(report)
+        path = report.save_baseline!
+        puts "  Baseline saved: #{path}"
+      end
+
       def task_prerequisites
         Rake::Task.task_defined?(:environment) ? [:environment] : []
       end

data/lib/ruby_llm/contract/rspec/helpers.rb

@@ -10,18 +10,38 @@ module RubyLLM
         #   result = ClassifyTicket.run("test")
         #   result.parsed_output  # => {priority: "high"}
         #
-        # For multiple sequential responses:
-        #   stub_step(ClassifyTicket, responses: [{ a: 1 }, { a: 2 }])
+        # Only affects the specified step — other steps are not affected.
         #
         def stub_step(step_class, response: nil, responses: nil)
-          adapter = if responses
-                      Adapters::Test.new(responses: responses.map { |r| r.is_a?(String) ? r : r.to_json })
-                    else
-                      content = response.is_a?(String) ? response : response.to_json
-                      Adapters::Test.new(response: content)
-                    end
+          adapter = build_test_adapter(response: response, responses: responses)
+          allow(step_class).to receive(:run).and_wrap_original do |original, input, **kwargs|
+            context = (kwargs[:context] || {}).merge(adapter: adapter)
+            original.call(input, context: context)
+          end
+        end
+
+        # Set a global test adapter for ALL steps.
+        #
+        #   stub_all_steps(response: { default: true })
+        #
+        def stub_all_steps(response: nil, responses: nil)
+          adapter = build_test_adapter(response: response, responses: responses)
           RubyLLM::Contract.configure { |c| c.default_adapter = adapter }
         end
+
+        private
+
+        def build_test_adapter(response: nil, responses: nil)
+          if responses
+            Adapters::Test.new(responses: responses.map { |r| normalize_test_response(r) })
+          else
+            Adapters::Test.new(response: normalize_test_response(response))
+          end
+        end
+
+        def normalize_test_response(value)
+          value
+        end
       end
     end
   end

data/lib/ruby_llm/contract/rspec/pass_eval.rb

@@ -64,12 +64,18 @@ RSpec::Matchers.define :pass_eval do |eval_name|
     @maximum_cost = cost
   end
 
+  chain :without_regressions do
+    @check_regressions = true
+  end
+
   match do |step_or_pipeline|
     @eval_name = eval_name
     @context ||= {}
     @minimum_score ||= nil
     @maximum_cost ||= nil
+    @check_regressions ||= false
     @error = nil
+    @diff = nil
     @report = step_or_pipeline.run_eval(eval_name, context: @context)
 
     score_ok = if @minimum_score
@@ -80,14 +86,29 @@ RSpec::Matchers.define :pass_eval do |eval_name|
 
     cost_ok = @maximum_cost ? @report.total_cost <= @maximum_cost : true
 
-    score_ok && cost_ok
+    regression_ok = if @check_regressions && @report.baseline_exists?
+                      @diff = @report.compare_with_baseline
+                      !@diff.regressed?
+                    else
+                      true
+                    end
+
+    score_ok && cost_ok && regression_ok
   rescue StandardError => e
     @error = e
     false
   end
 
   failure_message do
-    format_failure_message(@eval_name, @error, @report, @minimum_score, @maximum_cost)
+    msg = format_failure_message(@eval_name, @error, @report, @minimum_score, @maximum_cost)
+    if @diff&.regressed?
+      msg += "\n\nRegressions from baseline:\n"
+      @diff.regressions.each do |r|
+        msg += "  #{r[:case]}: was PASS, now FAIL — #{r[:detail]}\n"
+      end
+      msg += "  Score delta: #{@diff.score_delta}"
+    end
+    msg
   end
 
   failure_message_when_negated do

data/lib/ruby_llm/contract/step/base.rb

@@ -58,18 +58,23 @@ module RubyLLM
             end
           end
 
-          KNOWN_CONTEXT_KEYS = %i[adapter model temperature max_tokens schema provider assume_model_exists].freeze
+          KNOWN_CONTEXT_KEYS = %i[adapter model temperature max_tokens provider assume_model_exists].freeze
 
           def run(input, context: {})
+            context = (context || {}).transform_keys { |k| k.respond_to?(:to_sym) ? k.to_sym : k }
             warn_unknown_context_keys(context)
             adapter = resolve_adapter(context)
             default_model = context[:model] || model || RubyLLM::Contract.configuration.default_model
             policy = retry_policy
 
+            ctx_temp = context[:temperature]
+            extra = context.slice(:provider, :assume_model_exists, :max_tokens)
             result = if policy
-                       run_with_retry(input, adapter: adapter, default_model: default_model, policy: policy)
+                       run_with_retry(input, adapter: adapter, default_model: default_model,
+                                      policy: policy, context_temperature: ctx_temp, extra_options: extra)
                      else
-                       run_once(input, adapter: adapter, model: default_model, context_temperature: context[:temperature])
+                       run_once(input, adapter: adapter, model: default_model,
+                                context_temperature: ctx_temp, extra_options: extra)
                      end
 
             invoke_around_call(input, result)
@@ -101,14 +106,14 @@ module RubyLLM
                                             "{ |c| c.default_adapter = ... } or pass context: { adapter: ... }"
           end
 
-          def run_once(input, adapter:, model:, context_temperature: nil)
+          def run_once(input, adapter:, model:, context_temperature: nil, extra_options: {})
             effective_temp = context_temperature || temperature
             Runner.new(
               input_type: input_type, output_type: output_type,
               prompt_block: prompt, contract_definition: effective_contract,
               adapter: adapter, model: model, output_schema: output_schema,
               max_output: max_output, max_input: max_input, max_cost: max_cost,
-              temperature: effective_temp
+              temperature: effective_temp, extra_options: extra_options
             ).call(input)
           rescue ArgumentError => e
             Result.new(status: :input_error, raw_output: nil, parsed_output: nil,

data/lib/ruby_llm/contract/step/dsl.rb

@@ -168,7 +168,7 @@ module RubyLLM
         end
 
         def retry_policy(models: nil, attempts: nil, retry_on: nil, &block)
-          if block || models || attempts
+          if block || models || attempts || retry_on
             return @retry_policy = RetryPolicy.new(models: models, attempts: attempts, retry_on: retry_on, &block)
           end
 

data/lib/ruby_llm/contract/step/limit_checker.rb

@@ -29,7 +29,7 @@ module RubyLLM
         end
 
         def append_cost_error(estimated, errors)
-          estimated_output = @max_output || 0
+          estimated_output = effective_max_output || 0
           estimated_cost = CostCalculator.calculate(
             model_name: @model,
             usage: { input_tokens: estimated, output_tokens: estimated_output }

data/lib/ruby_llm/contract/step/retry_executor.rb

@@ -8,12 +8,13 @@ module RubyLLM
       module RetryExecutor
         private
 
-        def run_with_retry(input, adapter:, default_model:, policy:)
+        def run_with_retry(input, adapter:, default_model:, policy:, context_temperature: nil, extra_options: {})
           all_attempts = []
 
           policy.max_attempts.times do |attempt_index|
             model = policy.model_for_attempt(attempt_index, default_model)
-            result = run_once(input, adapter: adapter, model: model)
+            result = run_once(input, adapter: adapter, model: model,
+                              context_temperature: context_temperature, extra_options: extra_options)
             all_attempts << { attempt: attempt_index + 1, model: model, result: result }
             break unless policy.retryable?(result)
           end

data/lib/ruby_llm/contract/step/retry_policy.rb

@@ -15,6 +15,7 @@ module RubyLLM
           if block
             @max_attempts = 1
             instance_eval(&block)
+            warn_no_retry! if @max_attempts == 1 && @models.empty?
           else
             apply_keywords(models: models, attempts: attempts, retry_on: retry_on)
           end
@@ -38,7 +39,7 @@ module RubyLLM
         end
 
         def retry_on(*statuses)
-          @retryable_statuses = statuses
+          @retryable_statuses = statuses.flatten
         end
 
         def retryable?(result)
@@ -65,6 +66,11 @@ module RubyLLM
           @retryable_statuses = Array(retry_on).dup if retry_on
         end
 
+        def warn_no_retry!
+          warn "[ruby_llm-contract] retry_policy has max_attempts=1 with no models. " \
+               "This means no actual retry will happen. Add `attempts 2` or `escalate %w[model1 model2]`."
+        end
+
         def validate_max_attempts!
           return if @max_attempts.is_a?(Integer) && @max_attempts >= 1
 

data/lib/ruby_llm/contract/step/runner.rb

@@ -8,7 +8,7 @@ module RubyLLM
 
         def initialize(input_type:, output_type:, prompt_block:, contract_definition:,
                        adapter:, model:, output_schema: nil, max_output: nil,
-                       max_input: nil, max_cost: nil, temperature: nil)
+                       max_input: nil, max_cost: nil, temperature: nil, extra_options: {})
           @input_type = input_type
           @output_type = output_type
           @prompt_block = prompt_block
@@ -20,6 +20,7 @@ module RubyLLM
           @max_input = max_input
           @max_cost = max_cost
           @temperature = temperature
+          @extra_options = extra_options
         end
 
         def call(input)
@@ -82,13 +83,20 @@ module RubyLLM
         end
 
         def build_adapter_options
+          effective_max_tokens = @extra_options[:max_tokens] || @max_output
+
           { model: @model }.tap do |opts|
             opts[:schema] = @output_schema if @output_schema
-            opts[:max_tokens] = @max_output if @max_output
+            opts[:max_tokens] = effective_max_tokens if effective_max_tokens
             opts[:temperature] = @temperature if @temperature
+            @extra_options.each { |k, v| opts[k] = v unless opts.key?(k) }
           end
         end
 
+        def effective_max_output
+          @extra_options[:max_tokens] || @max_output
+        end
+
         def build_error_result(error_result, messages)
           Result.new(
             status: error_result.status,

data/lib/ruby_llm/contract/step/trace.rb

@@ -5,15 +5,16 @@ module RubyLLM
     module Step
       class Trace
         include Concerns::TraceEquality
+        include Concerns::DeepFreeze
 
         attr_reader :messages, :model, :latency_ms, :usage, :attempts, :cost
 
         def initialize(messages: nil, model: nil, latency_ms: nil, usage: nil, attempts: nil, cost: nil)
-          @messages = messages
-          @model = model
+          @messages = deep_dup_freeze(messages)
+          @model = model.frozen? ? model : model&.dup&.freeze
           @latency_ms = latency_ms
-          @usage = usage
-          @attempts = attempts
+          @usage = deep_dup_freeze(usage)
+          @attempts = deep_dup_freeze(attempts)
           @cost = cost || CostCalculator.calculate(model_name: model, usage: usage)
           freeze
         end

data/lib/ruby_llm/contract/version.rb

@@ -2,6 +2,6 @@
 
 module RubyLLM
   module Contract
-    VERSION = "0.2.3"
+    VERSION = "0.3.6"
   end
 end