ruby_llm-contract 0.4.2 → 0.5.0

data/lib/ruby_llm/contract/rspec/helpers.rb

@@ -12,11 +12,77 @@ module RubyLLM
         #
         # Only affects the specified step — other steps are not affected.
         #
-        def stub_step(step_class, response: nil, responses: nil)
+        # With a block, the stub is scoped — cleaned up after the block:
+        #
+        #   stub_step(ClassifyTicket, response: data) do
+        #     # only stubbed inside this block
+        #   end
+        #   # ClassifyTicket no longer stubbed
+        #
+        # Without a block, the stub lives until the RSpec example ends.
+        #
+        def stub_step(step_class, response: nil, responses: nil, &block)
           adapter = build_test_adapter(response: response, responses: responses)
-          allow(step_class).to receive(:run).and_wrap_original do |original, input, **kwargs|
-            context = (kwargs[:context] || {}).merge(adapter: adapter)
-            original.call(input, context: context)
+
+          if block
+            # Block form: use thread-local overrides with save/restore for real scoping
+            overrides = RubyLLM::Contract.step_adapter_overrides
+            previous = overrides[step_class]
+            overrides[step_class] = adapter
+            begin
+              yield
+            ensure
+              if previous
+                overrides[step_class] = previous
+              else
+                overrides.delete(step_class)
+              end
+            end
+          else
+            # Non-block: use RSpec allow (auto-cleaned after example)
+            allow(step_class).to receive(:run).and_wrap_original do |original, input, **kwargs|
+              context = kwargs[:context] || {}
+              unless context.key?(:adapter) || context.key?("adapter")
+                context = context.merge(adapter: adapter)
+              end
+              original.call(input, context: context)
+            end
+          end
+        end
+
+        # Stub multiple steps at once with different responses.
+        # Takes a hash of step_class => options. Requires a block.
+        #
+        #   stub_steps(
+        #     ClassifyTicket => { response: { priority: "high" } },
+        #     RouteToTeam => { response: { team: "billing" } }
+        #   ) do
+        #     result = TicketPipeline.run("test")
+        #   end
+        #
+        def stub_steps(stubs, &block)
+          raise ArgumentError, "stub_steps requires a block" unless block
+
+          overrides = RubyLLM::Contract.step_adapter_overrides
+          previous = {}
+
+          stubs.each do |step_class, opts|
+            opts = opts.transform_keys(&:to_sym)
+            adapter = build_test_adapter(**opts)
+            previous[step_class] = overrides[step_class]
+            overrides[step_class] = adapter
+          end
+
+          begin
+            yield
+          ensure
+            stubs.each_key do |step_class|
+              if previous[step_class]
+                overrides[step_class] = previous[step_class]
+              else
+                overrides.delete(step_class)
+              end
+            end
           end
         end
 
@@ -24,9 +90,28 @@ module RubyLLM
         #
         #   stub_all_steps(response: { default: true })
         #
-        def stub_all_steps(response: nil, responses: nil)
+        # Supports an optional block form — the previous adapter is restored
+        # after the block returns (even if it raises):
+        #
+        #   stub_all_steps(response: { default: true }) do
+        #     # all steps use test adapter
+        #   end
+        #   # original adapter restored
+        #
+        def stub_all_steps(response: nil, responses: nil, &block)
           adapter = build_test_adapter(response: response, responses: responses)
-          RubyLLM::Contract.configure { |c| c.default_adapter = adapter }
+
+          if block
+            previous = RubyLLM::Contract.configuration.default_adapter
+            begin
+              RubyLLM::Contract.configuration.default_adapter = adapter
+              yield
+            ensure
+              RubyLLM::Contract.configuration.default_adapter = previous
+            end
+          else
+            RubyLLM::Contract.configure { |c| c.default_adapter = adapter }
+          end
         end
 
         private

data/lib/ruby_llm/contract/rspec/pass_eval.rb

@@ -68,15 +68,28 @@ RSpec::Matchers.define :pass_eval do |eval_name|
     @check_regressions = true
   end
 
+  chain :compared_with do |other_step|
+    @comparison_step = other_step
+    @check_regressions = true # compared_with implies regression check
+  end
+
   match do |step_or_pipeline|
     @eval_name = eval_name
     @context ||= {}
     @minimum_score ||= nil
     @maximum_cost ||= nil
     @check_regressions ||= false
+    @comparison_step ||= nil
     @error = nil
     @diff = nil
-    @report = step_or_pipeline.run_eval(eval_name, context: @context)
+    @prompt_diff = nil
+
+    if @comparison_step && @check_regressions
+      @prompt_diff = step_or_pipeline.compare_with(@comparison_step, eval: eval_name, context: @context)
+      @report = @prompt_diff.candidate_report
+    else
+      @report = step_or_pipeline.run_eval(eval_name, context: @context)
+    end
 
     score_ok = if @minimum_score
                  @report.score >= @minimum_score
@@ -86,7 +99,9 @@ RSpec::Matchers.define :pass_eval do |eval_name|
 
     cost_ok = @maximum_cost ? @report.total_cost <= @maximum_cost : true
 
-    regression_ok = if @check_regressions && @report.baseline_exists?
+    regression_ok = if @prompt_diff
+                      @prompt_diff.safe_to_switch?
+                    elsif @check_regressions && @report.baseline_exists?
                       @diff = @report.compare_with_baseline
                       !@diff.regressed?
                     else
@@ -100,11 +115,67 @@ RSpec::Matchers.define :pass_eval do |eval_name|
   end
 
   failure_message do
+    if @prompt_diff && !@prompt_diff.safe_to_switch?
+      msg = "expected #{@eval_name} eval to be safe to switch from baseline prompt\n"
+
+      # Check empty sides first — most fundamental problem
+      bl_empty = @prompt_diff.baseline_empty?
+      cd_empty = @prompt_diff.candidate_empty?
+      if bl_empty || cd_empty
+        msg += "  One side has no evaluated cases (all skipped or no adapter?)\n"
+        if sample_response_only_compare?
+          msg += "  compare_with ignores sample_response; pass model: or with_context(adapter: ...)\n"
+        end
+        msg += "  Candidate score: #{@prompt_diff.candidate_score}, Baseline score: #{@prompt_diff.baseline_score}"
+        next msg
+      end
+
+      # Check dataset comparability — names, inputs, AND expected must match
+      unless @prompt_diff.cases_comparable?
+        unless @prompt_diff.case_names_match?
+          mm = @prompt_diff.mismatched_cases
+          msg += "  Case set mismatch — candidate and baseline must have identical cases:\n"
+          mm[:only_in_baseline].each { |n| msg += "    only in baseline: #{n}\n" }
+          mm[:only_in_candidate].each { |n| msg += "    only in candidate: #{n}\n" }
+        end
+        @prompt_diff.input_mismatches.each do |m|
+          msg += "  Input mismatch for '#{m[:case]}' — same name but different inputs\n"
+        end
+        @prompt_diff.expected_mismatches.each do |m|
+          msg += "  Expected mismatch for '#{m[:case]}' — same name/input but different expected values\n"
+        end
+        next msg
+      end
+
+      # Check per-case score regressions (even if global average is flat)
+      if @prompt_diff.score_regressions.any?
+        msg += "  Per-case score regressions (#{@prompt_diff.score_regressions.length}):\n"
+        @prompt_diff.score_regressions.each do |r|
+          msg += "    #{r[:case]}: #{r[:baseline_score]} -> #{r[:candidate_score]} (#{r[:delta]})\n"
+        end
+        msg += "  Score delta: #{@prompt_diff.score_delta}"
+        next msg
+      end
+
+      # Check pass/fail regressions and removed cases
+      removed = @prompt_diff.removed_passing_cases
+      reg_count = @prompt_diff.regressions.length + removed.length
+      msg += "  Found #{reg_count} regression(s):\n"
+      @prompt_diff.regressions.each do |r|
+        msg += "    #{r[:case]}: was PASS, now FAIL -- #{r[:detail]}\n"
+      end
+      removed.each do |name|
+        msg += "    #{name}: REMOVED (was passing in baseline)\n"
+      end
+      msg += "  Score delta: #{@prompt_diff.score_delta}"
+      next msg
+    end
+
     msg = format_failure_message(@eval_name, @error, @report, @minimum_score, @maximum_cost)
     if @diff&.regressed?
       msg += "\n\nRegressions from baseline:\n"
       @diff.regressions.each do |r|
-        msg += "  #{r[:case]}: was PASS, now FAIL — #{r[:detail]}\n"
+        msg += "  #{r[:case]}: was PASS, now FAIL -- #{r[:detail]}\n"
       end
       msg += "  Score delta: #{@diff.score_delta}"
     end
@@ -114,4 +185,14 @@ RSpec::Matchers.define :pass_eval do |eval_name|
   failure_message_when_negated do
     "expected #{@eval_name} eval NOT to pass, but it passed with score: #{@report.score.round(2)}"
   end
+
+  def sample_response_only_compare?
+    return false unless @comparison_step
+    return false if @context[:adapter] || @context[:model]
+
+    defn = @comparison_step.send(:all_eval_definitions)[@eval_name.to_s]
+    defn&.build_adapter
+  rescue StandardError
+    false
+  end
 end

data/lib/ruby_llm/contract/rspec.rb

@@ -8,4 +8,22 @@ require_relative "rspec/helpers"
 
 RSpec.configure do |config|
   config.include RubyLLM::Contract::RSpec::Helpers
+
+  # Auto-cleanup: snapshot adapter before each example, restore after.
+  # Prevents non-block stub_all_steps from leaking between examples.
+  config.around(:each) do |example|
+    original_adapter = RubyLLM::Contract.configuration.default_adapter
+    original_logger = RubyLLM::Contract.configuration.logger
+    original_eval_hosts = RubyLLM::Contract.eval_hosts.dup
+    original_overrides = RubyLLM::Contract.step_adapter_overrides.dup
+    begin
+      example.run
+    ensure
+      RubyLLM::Contract.configuration.default_adapter = original_adapter
+      RubyLLM::Contract.configuration.logger = original_logger
+      RubyLLM::Contract.reset_eval_hosts!
+      RubyLLM::Contract.eval_hosts.concat(original_eval_hosts)
+      RubyLLM::Contract.step_adapter_overrides.replace(original_overrides)
+    end
+  end
 end if defined?(::RSpec)

data/lib/ruby_llm/contract/step/adapter_caller.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Step
+      class AdapterCaller
+        def initialize(adapter:, adapter_options:)
+          @adapter = adapter
+          @adapter_options = adapter_options
+        end
+
+        def call(messages)
+          start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          response = @adapter.call(messages: messages, **@adapter_options)
+          latency_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
+          [response, latency_ms]
+        rescue StandardError => error
+          [Result.new(status: :adapter_error, raw_output: nil, parsed_output: nil, validation_errors: [error.message]), 0]
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/step/base.rb

@@ -4,6 +4,8 @@ module RubyLLM
   module Contract
     module Step
       class Base
+        DEFAULT_OUTPUT_TOKENS = 256
+
         def self.inherited(subclass)
           super
           Contract.register_eval_host(subclass) if respond_to?(:eval_defined?) && eval_defined?
@@ -15,30 +17,23 @@ module RubyLLM
           include Dsl
 
           def eval_case(input:, expected: nil, expected_traits: nil, evaluator: nil, context: {})
-            dataset = Eval::Dataset.define("single_case") do
-              add_case("inline", input: input, expected: expected,
-                                 expected_traits: expected_traits, evaluator: evaluator)
-            end
-            report = Eval::Runner.run(step: self, dataset: dataset, context: context)
-            report.results.first
+            Eval::Runner.run(step: self, dataset: inline_dataset(input, expected, expected_traits, evaluator),
+                             context: context).results.first
           end
 
           def estimate_cost(input:, model: nil)
-            model_name = model || RubyLLM::Contract.configuration.default_model
-            messages = build_messages(input)
-            input_tokens = TokenEstimator.estimate(messages)
-            output_tokens = max_output || 256 # conservative default
-
+            model_name = estimated_model_name(model)
             model_info = CostCalculator.send(:find_model, model_name)
             return nil unless model_info
 
-            estimated = CostCalculator.send(:compute_cost, model_info,
-                                            { input_tokens: input_tokens, output_tokens: output_tokens })
+            input_tokens = TokenEstimator.estimate(build_messages(input))
+            output_tokens = max_output || DEFAULT_OUTPUT_TOKENS
+
             {
               model: model_name,
               input_tokens: input_tokens,
               output_tokens_estimate: output_tokens,
-              estimated_cost: estimated
+              estimated_cost: estimated_cost_for(model_info, input_tokens, output_tokens)
             }
           end
 
@@ -46,15 +41,11 @@ module RubyLLM
             defn = send(:all_eval_definitions)[eval_name.to_s]
             raise ArgumentError, "No eval '#{eval_name}' defined" unless defn
 
-            model_list = models || [RubyLLM::Contract.configuration.default_model].compact
+            model_list = models || [estimated_model_name].compact
             cases = defn.build_dataset.cases
 
             model_list.each_with_object({}) do |model_name, result|
-              per_case = cases.sum do |c|
-                est = estimate_cost(input: c.input, model: model_name)
-                est ? est[:estimated_cost] : 0.0
-              end
-              result[model_name] = per_case.round(6)
+              result[model_name] = estimate_eval_cost_for_model(cases, model_name)
             end
           end
 
@@ -65,20 +56,8 @@ module RubyLLM
           def run(input, context: {})
             context = safe_context(context)
             warn_unknown_context_keys(context)
-            adapter = resolve_adapter(context)
-            default_model = context[:model] || model || RubyLLM::Contract.configuration.default_model
-            policy = retry_policy
-
-            ctx_temp = context[:temperature]
-            extra = context.slice(:provider, :assume_model_exists, :max_tokens)
-            result = if policy
-                       run_with_retry(input, adapter: adapter, default_model: default_model,
-                                      policy: policy, context_temperature: ctx_temp, extra_options: extra)
-                     else
-                       run_once(input, adapter: adapter, model: default_model,
-                                context_temperature: ctx_temp, extra_options: extra)
-                     end
 
+            result = dispatch_run(input, context)
             log_result(result)
             invoke_around_call(input, result)
           end
@@ -87,13 +66,43 @@ module RubyLLM
             dynamic = prompt.arity >= 1
             builder_input = dynamic ? input : Prompt::Builder::NOT_PROVIDED
             ast = Prompt::Builder.build(input: builder_input, &prompt)
-            variables = dynamic ? {} : { input: input }
-            variables.merge!(input.transform_keys(&:to_sym)) if !dynamic && input.is_a?(Hash)
-            Prompt::Renderer.render(ast, variables: variables)
+            Prompt::Renderer.render(ast, variables: prompt_variables(input, dynamic))
           end
 
           private
 
+          def inline_dataset(input, expected, expected_traits, evaluator)
+            Eval::Dataset.define("single_case") do
+              add_case("inline", input: input, expected: expected,
+                                 expected_traits: expected_traits, evaluator: evaluator)
+            end
+          end
+
+          def estimated_model_name(model = nil)
+            model || (self.model if respond_to?(:model)) || RubyLLM::Contract.configuration.default_model
+          end
+
+          def estimated_cost_for(model_info, input_tokens, output_tokens)
+            CostCalculator.send(
+              :compute_cost,
+              model_info,
+              { input_tokens: input_tokens, output_tokens: output_tokens }
+            )
+          end
+
+          def estimate_eval_cost_for_model(cases, model_name)
+            cases.sum do |test_case|
+              estimate = estimate_cost(input: test_case.input, model: model_name)
+              estimate ? estimate[:estimated_cost] : 0.0
+            end.round(6)
+          end
+
+          def prompt_variables(input, dynamic)
+            variables = dynamic ? {} : { input: input }
+            variables.merge!(input.transform_keys(&:to_sym)) if !dynamic && input.is_a?(Hash)
+            variables
+          end
+
           def warn_unknown_context_keys(context)
             unknown = context.keys - KNOWN_CONTEXT_KEYS
             return if unknown.empty?
@@ -102,6 +111,39 @@ module RubyLLM
                  "Known keys: #{KNOWN_CONTEXT_KEYS.inspect}"
           end
 
+          def dispatch_run(input, context)
+            adapter = resolve_adapter(context)
+            runtime = runtime_settings(context)
+
+            if runtime[:policy]
+              run_with_retry(
+                input,
+                adapter: adapter,
+                default_model: runtime[:model],
+                policy: runtime[:policy],
+                context_temperature: runtime[:temperature],
+                extra_options: runtime[:extra_options]
+              )
+            else
+              run_once(
+                input,
+                adapter: adapter,
+                model: runtime[:model],
+                context_temperature: runtime[:temperature],
+                extra_options: runtime[:extra_options]
+              )
+            end
+          end
+
+          def runtime_settings(context)
+            {
+              model: context[:model] || model || RubyLLM::Contract.configuration.default_model,
+              temperature: context[:temperature],
+              extra_options: context.slice(:provider, :assume_model_exists, :max_tokens),
+              policy: retry_policy
+            }
+          end
+
           def resolve_adapter(context)
             adapter = context[:adapter] || RubyLLM::Contract.configuration.default_adapter
             return adapter if adapter
@@ -117,7 +159,9 @@ module RubyLLM
               prompt_block: prompt, contract_definition: effective_contract,
               adapter: adapter, model: model, output_schema: output_schema,
               max_output: max_output, max_input: max_input, max_cost: max_cost,
-              temperature: effective_temp, extra_options: extra_options
+              on_unknown_pricing: on_unknown_pricing,
+              temperature: effective_temp, extra_options: extra_options,
+              observers: class_observers
             ).call(input)
           rescue ArgumentError => e
             Result.new(status: :input_error, raw_output: nil, parsed_output: nil,
@@ -135,6 +179,19 @@ module RubyLLM
                   "tokens=#{trace.usage&.dig(:input_tokens) || 0}+#{trace.usage&.dig(:output_tokens) || 0} " \
                   "cost=$#{format("%.6f", trace.cost || 0)}"
             logger.info(msg)
+
+            log_failed_observations(result, logger)
+          end
+
+          def log_failed_observations(result, logger)
+            failed = result.observations.select { |o| !o[:passed] }
+            return if failed.empty?
+
+            failed.each do |obs|
+              msg = "[ruby_llm-contract] #{name || self} observation failed: #{obs[:description]}"
+              msg += " (#{obs[:error]})" if obs[:error]
+              logger.warn(msg)
+            end
           end
 
           def invoke_around_call(input, result)

data/lib/ruby_llm/contract/step/dsl.rb

@@ -79,6 +79,16 @@ module RubyLLM
           inherited + own
         end
 
+        def observe(description, &block)
+          (@class_observers ||= []) << Invariant.new(description, block)
+        end
+
+        def class_observers
+          own = defined?(@class_observers) ? @class_observers : []
+          inherited = superclass.respond_to?(:class_observers) ? superclass.class_observers : []
+          inherited + own
+        end
+
         def max_output(tokens = nil)
           if tokens
             unless tokens.is_a?(Numeric) && tokens.positive?
@@ -111,48 +121,83 @@ module RubyLLM
           end
         end
 
-        def max_cost(amount = nil)
+        def max_cost(amount = nil, on_unknown_pricing: nil)
+          if amount == :default
+            @max_cost = nil
+            @max_cost_explicitly_unset = true
+            @on_unknown_pricing = nil
+            return nil
+          end
+
           if amount
             unless amount.is_a?(Numeric) && amount.positive?
               raise ArgumentError, "max_cost must be positive, got #{amount}"
             end
 
-            return @max_cost = amount
+            if on_unknown_pricing && !%i[refuse warn].include?(on_unknown_pricing)
+              raise ArgumentError, "on_unknown_pricing must be :refuse or :warn, got #{on_unknown_pricing.inspect}"
+            end
+
+            @max_cost_explicitly_unset = false
+            @max_cost = amount
+            @on_unknown_pricing = on_unknown_pricing || :refuse
+            return @max_cost
           end
 
-          if defined?(@max_cost)
-            @max_cost
-          elsif superclass.respond_to?(:max_cost)
-            superclass.max_cost
+          return @max_cost if defined?(@max_cost) && !@max_cost_explicitly_unset
+          return nil if @max_cost_explicitly_unset
+
+          superclass.max_cost if superclass.respond_to?(:max_cost)
+        end
+
+        def on_unknown_pricing
+          if defined?(@on_unknown_pricing)
+            @on_unknown_pricing
+          elsif superclass.respond_to?(:on_unknown_pricing)
+            superclass.on_unknown_pricing
+          else
+            :refuse
           end
         end
 
         def model(name = nil)
+          if name == :default
+            @model = nil
+            @model_explicitly_unset = true
+            return nil
+          end
+
           if name
+            @model_explicitly_unset = false
             return @model = name
           end
 
-          if defined?(@model)
-            @model
-          elsif superclass.respond_to?(:model)
-            superclass.model
-          end
+          return @model if defined?(@model) && !@model_explicitly_unset
+          return nil if @model_explicitly_unset
+
+          superclass.model if superclass.respond_to?(:model)
         end
 
         def temperature(value = nil)
+          if value == :default
+            @temperature = nil
+            @temperature_explicitly_unset = true
+            return nil
+          end
+
           if value
             unless value.is_a?(Numeric) && value >= 0 && value <= 2
               raise ArgumentError, "temperature must be 0.0-2.0, got #{value}"
             end
 
+            @temperature_explicitly_unset = false
             return @temperature = value
           end
 
-          if defined?(@temperature)
-            @temperature
-          elsif superclass.respond_to?(:temperature)
-            superclass.temperature
-          end
+          return @temperature if defined?(@temperature) && !@temperature_explicitly_unset
+          return nil if @temperature_explicitly_unset
+
+          superclass.temperature if superclass.respond_to?(:temperature)
         end
 
         def around_call(&block)

data/lib/ruby_llm/contract/step/input_validator.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Step
+      class InputValidator
+        def initialize(input_type:)
+          @input_type = input_type
+        end
+
+        def call(input)
+          validate(input)
+          nil
+        rescue Dry::Types::CoercionError, TypeError, ArgumentError => error
+          Result.new(status: :input_error, raw_output: nil, parsed_output: nil, validation_errors: [error.message])
+        end
+
+        private
+
+        def validate(input)
+          if ruby_class_input?
+            raise TypeError, "#{input.inspect} is not a #{@input_type}" unless input.is_a?(@input_type)
+          else
+            @input_type[input]
+          end
+        end
+
+        def ruby_class_input?
+          @input_type.is_a?(Class) && !@input_type.respond_to?(:[])
+        end
+      end
+    end
+  end
+end