ruby_llm-contract 1.0.0 → 1.1.1

data/lib/ruby_llm/contract/rspec/pass_eval.rb

@@ -64,6 +64,10 @@ RSpec::Matchers.define :pass_eval do |eval_name|
     @maximum_cost = cost
   end
 
+  chain :on_unknown_pricing do |mode|
+    @on_unknown_pricing = RubyLLM::Contract::UnknownPolicy.validate!("on_unknown_pricing", mode)
+  end
+
   chain :without_regressions do
     @check_regressions = true
   end
@@ -78,6 +82,9 @@ RSpec::Matchers.define :pass_eval do |eval_name|
     @context ||= {}
     @minimum_score ||= nil
     @maximum_cost ||= nil
+    @on_unknown_pricing ||= RubyLLM::Contract::UnknownPolicy::DEFAULT
+    @unknown_cost_failure = nil
+    @unknown_cost_only = false
     @check_regressions ||= false
     @comparison_step ||= nil
     @error = nil
@@ -97,7 +104,11 @@ RSpec::Matchers.define :pass_eval do |eval_name|
                  @report.passed?
                end
 
-    cost_ok = @maximum_cost ? @report.total_cost <= @maximum_cost : true
+    cost_ok = true
+    if @maximum_cost
+      @unknown_cost_failure = RubyLLM::Contract::Eval::UnknownCostGate.check([@report], mode: @on_unknown_pricing)
+      cost_ok = @report.total_cost <= @maximum_cost && @unknown_cost_failure.nil?
+    end
 
     regression_ok = if @prompt_diff
                       @prompt_diff.safe_to_switch?
@@ -108,6 +119,7 @@ RSpec::Matchers.define :pass_eval do |eval_name|
                       true
                     end
 
+    @unknown_cost_only = score_ok && regression_ok && @report.total_cost <= @maximum_cost if @unknown_cost_failure
     score_ok && cost_ok && regression_ok
   rescue StandardError => e
     @error = e
@@ -115,71 +127,87 @@ RSpec::Matchers.define :pass_eval do |eval_name|
   end
 
   failure_message do
-    if @prompt_diff && !@prompt_diff.safe_to_switch?
-      msg = "expected #{@eval_name} eval to be safe to switch from baseline prompt\n"
-
-      # Check empty sides first — most fundamental problem
-      bl_empty = @prompt_diff.baseline_empty?
-      cd_empty = @prompt_diff.candidate_empty?
-      if bl_empty || cd_empty
-        msg += "  One side has no evaluated cases (all skipped or no adapter?)\n"
-        if sample_response_only_compare?
-          msg += "  compare_with ignores sample_response; pass model: or with_context(adapter: ...)\n"
+    other_failure_message = lambda do
+      if @prompt_diff && !@prompt_diff.safe_to_switch?
+        msg = "expected #{@eval_name} eval to be safe to switch from baseline prompt\n"
+
+        # Check empty sides first — most fundamental problem
+        bl_empty = @prompt_diff.baseline_empty?
+        cd_empty = @prompt_diff.candidate_empty?
+        if bl_empty || cd_empty
+          msg += "  One side has no evaluated cases (all skipped or no adapter?)\n"
+          if sample_response_only_compare?
+            msg += "  compare_with ignores sample_response; pass model: or with_context(adapter: ...)\n"
+          end
+          msg += "  Candidate score: #{@prompt_diff.candidate_score}, Baseline score: #{@prompt_diff.baseline_score}"
+          next msg
         end
-        msg += "  Candidate score: #{@prompt_diff.candidate_score}, Baseline score: #{@prompt_diff.baseline_score}"
-        next msg
-      end
 
-      # Check dataset comparability — names, inputs, AND expected must match
-      unless @prompt_diff.cases_comparable?
-        unless @prompt_diff.case_names_match?
-          mm = @prompt_diff.mismatched_cases
-          msg += "  Case set mismatch — candidate and baseline must have identical cases:\n"
-          mm[:only_in_baseline].each { |n| msg += "    only in baseline: #{n}\n" }
-          mm[:only_in_candidate].each { |n| msg += "    only in candidate: #{n}\n" }
-        end
-        @prompt_diff.input_mismatches.each do |m|
-          msg += "  Input mismatch for '#{m[:case]}' — same name but different inputs\n"
+        # Check dataset comparability — names, inputs, AND expected must match
+        unless @prompt_diff.cases_comparable?
+          unless @prompt_diff.case_names_match?
+            mm = @prompt_diff.mismatched_cases
+            msg += "  Case set mismatch — candidate and baseline must have identical cases:\n"
+            mm[:only_in_baseline].each { |n| msg += "    only in baseline: #{n}\n" }
+            mm[:only_in_candidate].each { |n| msg += "    only in candidate: #{n}\n" }
+          end
+          @prompt_diff.input_mismatches.each do |m|
+            msg += "  Input mismatch for '#{m[:case]}' — same name but different inputs\n"
+          end
+          @prompt_diff.expected_mismatches.each do |m|
+            msg += "  Expected mismatch for '#{m[:case]}' — same name/input but different expected values\n"
+          end
+          next msg
         end
-        @prompt_diff.expected_mismatches.each do |m|
-          msg += "  Expected mismatch for '#{m[:case]}' — same name/input but different expected values\n"
+
+        # Check per-case score regressions (even if global average is flat)
+        if @prompt_diff.score_regressions.any?
+          msg += "  Per-case score regressions (#{@prompt_diff.score_regressions.length}):\n"
+          @prompt_diff.score_regressions.each do |r|
+            msg += "    #{r[:case]}: #{r[:baseline_score]} -> #{r[:candidate_score]} (#{r[:delta]})\n"
+          end
+          msg += "  Score delta: #{@prompt_diff.score_delta}"
+          next msg
         end
-        next msg
-      end
 
-      # Check per-case score regressions (even if global average is flat)
-      if @prompt_diff.score_regressions.any?
-        msg += "  Per-case score regressions (#{@prompt_diff.score_regressions.length}):\n"
-        @prompt_diff.score_regressions.each do |r|
-          msg += "    #{r[:case]}: #{r[:baseline_score]} -> #{r[:candidate_score]} (#{r[:delta]})\n"
+        # Check pass/fail regressions and removed cases
+        removed = @prompt_diff.removed_passing_cases
+        reg_count = @prompt_diff.regressions.length + removed.length
+        msg += "  Found #{reg_count} regression(s):\n"
+        @prompt_diff.regressions.each do |r|
+          msg += "    #{r[:case]}: was PASS, now FAIL -- #{r[:detail]}\n"
+        end
+        removed.each do |name|
+          msg += "    #{name}: REMOVED (was passing in baseline)\n"
         end
         msg += "  Score delta: #{@prompt_diff.score_delta}"
         next msg
       end
 
-      # Check pass/fail regressions and removed cases
-      removed = @prompt_diff.removed_passing_cases
-      reg_count = @prompt_diff.regressions.length + removed.length
-      msg += "  Found #{reg_count} regression(s):\n"
-      @prompt_diff.regressions.each do |r|
-        msg += "    #{r[:case]}: was PASS, now FAIL -- #{r[:detail]}\n"
-      end
-      removed.each do |name|
-        msg += "    #{name}: REMOVED (was passing in baseline)\n"
+      msg = format_failure_message(@eval_name, @error, @report, @minimum_score, @maximum_cost)
+      if @diff&.regressed?
+        msg += "\n\nRegressions from baseline:\n"
+        @diff.regressions.each do |r|
+          msg += "  #{r[:case]}: was PASS, now FAIL -- #{r[:detail]}\n"
+        end
+        @diff.skipped_passing_cases.each do |s|
+          msg += "  #{s[:case]}: was PASS, now SKIPPED -- #{s[:detail]}\n"
+        end
+        msg += "  Score delta: #{@diff.score_delta}"
       end
-      msg += "  Score delta: #{@prompt_diff.score_delta}"
-      next msg
+      msg
     end
 
-    msg = format_failure_message(@eval_name, @error, @report, @minimum_score, @maximum_cost)
-    if @diff&.regressed?
-      msg += "\n\nRegressions from baseline:\n"
-      @diff.regressions.each do |r|
-        msg += "  #{r[:case]}: was PASS, now FAIL -- #{r[:detail]}\n"
-      end
-      msg += "  Score delta: #{@diff.score_delta}"
+    # Cost first, as in the rake suite gate: an unpriced total cannot be trusted.
+    # An exception outranks it, and any other failure is still listed.
+    if @unknown_cost_failure && @error.nil?
+      unknown_cost = "expected #{@eval_name} eval to stay within maximum cost, but #{@unknown_cost_failure}"
+      next unknown_cost if @unknown_cost_only
+
+      next "#{unknown_cost}\n\n#{other_failure_message.call}"
     end
-    msg
+
+    other_failure_message.call
   end
 
   failure_message_when_negated do

data/lib/ruby_llm/contract/step/base.rb

@@ -90,8 +90,10 @@ module RubyLLM
             ).call
           end
 
-          KNOWN_CONTEXT_KEYS = %i[adapter model temperature max_tokens provider assume_model_exists
-                                  reasoning_effort retry_policy_override attachment].freeze
+          # Forwarded to the adapter as-is. A key known but not forwarded would be
+          # silently dropped, so the known list is built from this one.
+          ADAPTER_CONTEXT_KEYS = %i[provider assume_model_exists max_tokens reasoning_effort attachment].freeze
+          KNOWN_CONTEXT_KEYS = (%i[adapter model temperature retry_policy_override] + ADAPTER_CONTEXT_KEYS).freeze
 
           include Concerns::ContextHelpers
 
@@ -133,7 +135,7 @@ module RubyLLM
             estimate = attachment_token_estimate if respond_to?(:attachment_token_estimate)
             return [estimate, false] unless estimate.nil?
 
-            mode = respond_to?(:on_unknown_attachment_size) ? on_unknown_attachment_size : :refuse
+            mode = respond_to?(:on_unknown_attachment_size) ? on_unknown_attachment_size : UnknownPolicy::DEFAULT
             if mode == :warn
               warn "[ruby_llm-contract] attachment present but attachment_token_estimate not " \
                    "declared on #{name || self} — estimate_cost proceeds without attachment cost"
@@ -193,7 +195,7 @@ module RubyLLM
 
           def runtime_settings(context)
             policy = context.key?(:retry_policy_override) ? context[:retry_policy_override] : retry_policy
-            extra = context.slice(:provider, :assume_model_exists, :max_tokens, :reasoning_effort, :attachment)
+            extra = context.slice(*ADAPTER_CONTEXT_KEYS)
 
             # Always pass the class-level `thinking` config to the adapter when
             # set, so fields like `budget` survive a per-call `reasoning_effort`

data/lib/ruby_llm/contract/step/dsl.rb

@@ -151,12 +151,10 @@ module RubyLLM
           if amount
             validate_positive!("max_cost", amount)
 
-            if on_unknown_pricing && !%i[refuse warn].include?(on_unknown_pricing)
-              raise ArgumentError, "on_unknown_pricing must be :refuse or :warn, got #{on_unknown_pricing.inspect}"
-            end
+            UnknownPolicy.validate!("on_unknown_pricing", on_unknown_pricing) if on_unknown_pricing
 
             @max_cost = amount
-            @on_unknown_pricing = on_unknown_pricing || :refuse
+            @on_unknown_pricing = on_unknown_pricing || UnknownPolicy::DEFAULT
             return @max_cost
           end
 
@@ -164,7 +162,7 @@ module RubyLLM
         end
 
         def on_unknown_pricing
-          inherited_value(:on_unknown_pricing) || :refuse
+          inherited_value(:on_unknown_pricing) || UnknownPolicy::DEFAULT
         end
 
         def attachment_token_estimate(n = nil)
@@ -182,16 +180,9 @@ module RubyLLM
         end
 
         def on_unknown_attachment_size(mode = nil)
-          if mode
-            unless %i[refuse warn].include?(mode)
-              raise ArgumentError,
-                    "on_unknown_attachment_size must be :refuse or :warn, got #{mode.inspect}"
-            end
-
-            return @on_unknown_attachment_size = mode
-          end
+          return @on_unknown_attachment_size = UnknownPolicy.validate!("on_unknown_attachment_size", mode) if mode
 
-          inherited_value(:on_unknown_attachment_size) || :refuse
+          inherited_value(:on_unknown_attachment_size) || UnknownPolicy::DEFAULT
         end
 
         def model(name = nil)

data/lib/ruby_llm/contract/step/runner_config.rb

@@ -28,8 +28,8 @@ module RubyLLM
         def self.build(input_type:, output_type:, prompt_block:, contract_definition:,
                        adapter:, model:,
                        output_schema: nil, max_output: nil,
-                       max_input: nil, max_cost: nil, on_unknown_pricing: :refuse,
-                       attachment_token_estimate: nil, on_unknown_attachment_size: :refuse,
+                       max_input: nil, max_cost: nil, on_unknown_pricing: UnknownPolicy::DEFAULT,
+                       attachment_token_estimate: nil, on_unknown_attachment_size: UnknownPolicy::DEFAULT,
                        temperature: nil, extra_options: {}, observers: [])
           new(
             input_type: input_type, output_type: output_type,

data/lib/ruby_llm/contract/step/trace.rb

@@ -50,6 +50,25 @@ module RubyLLM
           )
         end
 
+        # True when a call ran on a named model, used tokens, and still got no
+        # cost, so any total built from this trace undercounts. A zero-token call
+        # (Test adapter, sample_response) costs 0 whatever the pricing. A retried
+        # step is judged per attempt: a priced subtotal must not hide an unpriced
+        # attempt, and the merged trace may have re-priced on the last model.
+        def cost_unknown?
+          if @attempts.is_a?(Array) && !@attempts.empty?
+            @attempts.any? { |attempt| self.class.unpriced?(attempt[:model], attempt[:usage], attempt[:cost]) }
+          else
+            self.class.unpriced?(@model, @usage, @cost)
+          end
+        end
+
+        def self.unpriced?(model, usage, cost)
+          return false unless model && cost.nil? && usage.is_a?(Hash)
+
+          ((usage[:input_tokens] || 0) + (usage[:output_tokens] || 0)).positive?
+        end
+
         def to_h
           { messages: @messages, model: @model, latency_ms: @latency_ms,
             usage: @usage, attempts: @attempts, cost: @cost }.compact

data/lib/ruby_llm/contract/unknown_policy.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    # What a limit does when the number it needs is unknown: model pricing for
+    # max_cost/maximum_cost, attachment size for max_input. One vocabulary for
+    # every such option, step-level and eval-level, so a new mode or a changed
+    # default reaches all of them. Standalone: the rake task loads it before the
+    # rest of the gem to validate its settings at definition time.
+    module UnknownPolicy
+      MODES = %i[refuse warn].freeze
+      DEFAULT = :refuse
+
+      def self.validate!(option, mode)
+        return mode if MODES.include?(mode)
+
+        raise ArgumentError, "#{option} must be #{MODES.map(&:inspect).join(" or ")}, got #{mode.inspect}"
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/version.rb

@@ -2,6 +2,6 @@
 
 module RubyLLM
   module Contract
-    VERSION = "1.0.0"
+    VERSION = "1.1.1"
   end
 end

data/lib/ruby_llm/contract.rb

@@ -2,10 +2,19 @@
 
 require_relative "contract/version"
 require_relative "contract/errors"
+require_relative "contract/unknown_policy"
 require_relative "contract/types"
 
 module RubyLLM
   module Contract
+    # Rails dirs holding Step classes. Each one's eval/ subdir holds define_eval
+    # files, which define no constant, so Zeitwerk must ignore exactly those.
+    RAILS_CONTRACT_DIRS = %w[app/contracts app/steps].freeze
+    RAILS_EVAL_DIRS = RAILS_CONTRACT_DIRS.map { |dir| "#{dir}/eval" }.freeze
+
+    # Set while load_evals! runs: redefining an eval is then a reload, not a mistake.
+    RELOADING_KEY = :ruby_llm_contract_reloading
+
     class << self
       def configuration
         @configuration ||= Configuration.new
@@ -51,7 +60,7 @@ module RubyLLM
       def load_evals!(*dirs)
         dirs = dirs.flatten.compact
         if dirs.empty? && defined?(::Rails)
-          dirs = %w[app/steps/eval app/contracts/eval].filter_map do |path|
+          dirs = RAILS_EVAL_DIRS.filter_map do |path|
             full = ::Rails.root.join(path)
             full.to_s if full.exist?
           end
@@ -64,7 +73,7 @@ module RubyLLM
         eager_load_contract_dirs! if defined?(::Rails)
 
         # Clear file-sourced evals ONCE, then load ALL dirs.
-        Thread.current[:ruby_llm_contract_reloading] = true
+        Thread.current[RELOADING_KEY] = true
         eval_hosts.each do |host|
           host.clear_file_sourced_evals! if host.respond_to?(:clear_file_sourced_evals!)
         end
@@ -73,7 +82,11 @@ module RubyLLM
           Dir[File.join(d, "**", "*_eval.rb")].each { |f| load f }
         end
       ensure
-        Thread.current[:ruby_llm_contract_reloading] = false
+        Thread.current[RELOADING_KEY] = false
+      end
+
+      def reloading?
+        Thread.current[RELOADING_KEY] ? true : false
       end
 
       def normalize_candidate_config(entry)
@@ -111,7 +124,7 @@ module RubyLLM
       end
 
       def eager_load_contract_dirs!
-        %w[app/contracts app/steps].each do |path|
+        RAILS_CONTRACT_DIRS.each do |path|
           full = ::Rails.root.join(path)
           next unless full.exist?
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby_llm-contract
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Justyna
@@ -59,6 +59,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".ruby-version"
 - CHANGELOG.md
 - LICENSE
 - README.md
@@ -152,6 +153,7 @@ files:
 - lib/ruby_llm/contract/eval/step_expectation_applier.rb
 - lib/ruby_llm/contract/eval/step_result_normalizer.rb
 - lib/ruby_llm/contract/eval/trait_evaluator.rb
+- lib/ruby_llm/contract/eval/unknown_cost_gate.rb
 - lib/ruby_llm/contract/minitest.rb
 - lib/ruby_llm/contract/pipeline.rb
 - lib/ruby_llm/contract/pipeline/base.rb
@@ -191,6 +193,7 @@ files:
 - lib/ruby_llm/contract/step/trace.rb
 - lib/ruby_llm/contract/token_estimator.rb
 - lib/ruby_llm/contract/types.rb
+- lib/ruby_llm/contract/unknown_policy.rb
 - lib/ruby_llm/contract/version.rb
 - ruby_llm-contract.gemspec
 homepage: https://github.com/justi/ruby_llm-contract