ruby_llm-contract 1.1.0 → 1.1.2

data/lib/ruby_llm/contract/eval/eval_history.rb

@@ -69,7 +69,8 @@ module RubyLLM
 
           lines = ["#{runs.length} runs"]
           runs.last(5).each do |r|
-            lines << "  #{r[:date]} score=#{r[:score].round(2)} cost=$#{format("%.6f", r[:total_cost] || r[:cost] || 0)}"
+            cost = format("%.6f", r[:total_cost] || r[:cost] || 0)
+            lines << "  #{r[:date]} score=#{r[:score].round(2)} cost=$#{cost}#{" + unknown" if r[:cost_unknown]}"
           end
           lines.join("\n")
         end

data/lib/ruby_llm/contract/eval/evaluator/proc_evaluator.rb

@@ -6,6 +6,10 @@ module RubyLLM
       module Evaluator
         # Adapts custom Ruby callables to the EvaluationResult contract.
         class ProcEvaluator
+          # Report hides these under a failure: they repeat the PASS/FAIL label.
+          PASSED_DETAILS = "passed"
+          FAILED_DETAILS = "not passed"
+
           def initialize(callable)
             @callable = callable
           end
@@ -36,9 +40,9 @@ module RubyLLM
           def build_evaluation_result(result)
             case result
             when true
-              EvaluationResult.new(score: 1.0, passed: true, details: "passed")
+              EvaluationResult.new(score: 1.0, passed: true, details: PASSED_DETAILS)
             when false
-              EvaluationResult.new(score: 0.0, passed: false, details: "not passed")
+              EvaluationResult.new(score: 0.0, passed: false, details: FAILED_DETAILS)
             when Numeric
               EvaluationResult.new(score: result, passed: result >= 0.5, details: "custom score: #{result}")
             else

data/lib/ruby_llm/contract/eval/model_comparison.rb

@@ -36,8 +36,12 @@ module RubyLLM
           @reports[resolve_key(candidate)]&.total_cost
         end
 
+        # A report with unpriced cases has only a subtotal, so it cannot be
+        # called the cheapest.
         def best_for(min_score: 0.0)
-          eligible = @reports.select { |_, report| report.score > 0.0 && report.score >= min_score }
+          eligible = @reports.select do |_, report|
+            report.score > 0.0 && report.score >= min_score && !cost_unknown?(report)
+          end
           return nil if eligible.empty?
 
           eligible.min_by { |_, report| report.total_cost }&.first
@@ -45,6 +49,8 @@ module RubyLLM
 
         def cost_per_point
           @reports.transform_values do |report|
+            next Float::INFINITY if cost_unknown?(report)
+
             report.score.positive? ? report.total_cost / report.score : Float::INFINITY
           end
         end
@@ -163,6 +169,10 @@ module RubyLLM
           )
         end
 
+        def cost_unknown?(report)
+          report.respond_to?(:unknown_cost_results) && report.unknown_cost_results.any?
+        end
+
         def resolve_key(candidate)
           case candidate
           when String then candidate

data/lib/ruby_llm/contract/eval/prompt_diff_serializer.rb

@@ -5,7 +5,7 @@ module RubyLLM
     module Eval
       class PromptDiffSerializer
         def call(report)
-          report.results.reject { |result| result.step_status == :skipped }.map do |result|
+          report.results.reject { |result| result.step_status == CaseResult::SKIPPED_STATUS }.map do |result|
             {
               name: result.name,
               input: result.input,

data/lib/ruby_llm/contract/eval/recommender.rb

@@ -40,7 +40,7 @@ module RubyLLM
             report = @comparison.reports[label]
             next nil unless report
 
-            evaluated_count = report.results.count { |r| r.step_status != :skipped }
+            evaluated_count = report.results.count { |r| r.step_status != CaseResult::SKIPPED_STATUS }
             cases_count = [evaluated_count, 1].max
             cost_per_call = report.total_cost.to_f / cases_count
 
@@ -51,7 +51,8 @@ module RubyLLM
               cost_per_call: cost_per_call,
               latency: report.avg_latency_ms || Float::INFINITY,
               pass_rate_ratio: report.pass_rate_ratio,
-              total_cost: report.total_cost
+              total_cost: report.total_cost,
+              cost_unknown: unknown_cost?(report)
             }
           end
         end
@@ -114,9 +115,9 @@ module RubyLLM
 
           current_label = ModelComparison.candidate_label(@current_config)
           current_report = @comparison.reports[current_label]
-          return {} unless current_report
+          return {} if current_report.nil? || unknown_cost?(current_report)
 
-          current_evaluated = current_report.results.count { |r| r.step_status != :skipped }
+          current_evaluated = current_report.results.count { |r| r.step_status != CaseResult::SKIPPED_STATUS }
           current_cases = [current_evaluated, 1].max
           current_cost = current_report.total_cost.to_f / current_cases
           diff = current_cost - best[:cost_per_call]
@@ -125,8 +126,14 @@ module RubyLLM
           { per_call: diff.round(6), monthly_at: { 10_000 => (diff * 10_000).round(2) } }
         end
 
+        # A subtotal over unpriced cases is not a cost to rank by. Zero stays
+        # "unknown pricing" too: offline runs report $0 without any call.
         def cost_known?(scored_candidate)
-          scored_candidate[:cost_per_call]&.positive?
+          !scored_candidate[:cost_unknown] && scored_candidate[:cost_per_call]&.positive?
+        end
+
+        def unknown_cost?(report)
+          report.respond_to?(:unknown_cost_results) && report.unknown_cost_results.any?
         end
       end
     end

data/lib/ruby_llm/contract/eval/report.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "forwardable"
+require_relative "evaluator/proc_evaluator"
 
 module RubyLLM
   module Contract
@@ -10,9 +11,11 @@ module RubyLLM
 
         attr_reader :dataset_name, :results, :step_name
 
-        GENERIC_DETAILS = ["passed", "not passed"].freeze
+        GENERIC_DETAILS = [Evaluator::ProcEvaluator::PASSED_DETAILS, Evaluator::ProcEvaluator::FAILED_DETAILS].freeze
         HISTORY_DIR = ".eval_history"
+        HISTORY_EXT = "jsonl"
         BASELINE_DIR = ".eval_baselines"
+        BASELINE_EXT = "json"
 
         def_delegators :@stats, :score, :passed, :failed, :skipped, :failures, :pass_rate, :pass_rate_ratio,
                        :total_cost, :unknown_cost_results, :avg_latency_ms, :passed?,

data/lib/ruby_llm/contract/eval/report_stats.rb

@@ -23,7 +23,7 @@ module RubyLLM
         end
 
         def skipped
-          @results.count { |result| result.step_status == :skipped }
+          @results.count { |result| result.step_status == CaseResult::SKIPPED_STATUS }
         end
 
         def failures
@@ -63,7 +63,7 @@ module RubyLLM
         end
 
         def evaluated_results
-          @evaluated_results ||= @results.reject { |result| result.step_status == :skipped }
+          @evaluated_results ||= @results.reject { |result| result.step_status == CaseResult::SKIPPED_STATUS }
         end
 
         def evaluated_results_count
@@ -85,7 +85,7 @@ module RubyLLM
         def single_shot_cost
           return nil unless production_mode?
 
-          evaluated_results.sum { |r| first_attempt_cost(r) || r.cost || 0.0 }
+          evaluated_results.sum { |r| first_attempt_cost(r) || 0.0 }
         end
 
         def effective_cost
@@ -116,9 +116,14 @@ module RubyLLM
 
         private
 
+        # With attempts, only the first one's cost: falling back to the case
+        # cost when it is unpriced would report a later attempt's spend as the
+        # single-shot cost.
         def first_attempt_cost(result)
-          first = (result.attempts || []).first
-          first && first[:cost]
+          attempts = result.attempts || []
+          return result.cost if attempts.empty?
+
+          attempts.first[:cost]
         end
 
         def first_attempt_latency(result)

data/lib/ruby_llm/contract/eval/report_storage.rb

@@ -13,7 +13,7 @@ module RubyLLM
         end
 
         def save_history!(path: nil, model: nil, reasoning_effort: nil)
-          file = path || storage_path(Report::HISTORY_DIR, "jsonl", model: model, reasoning_effort: reasoning_effort)
+          file = path || storage_path(Report::HISTORY_DIR, Report::HISTORY_EXT, model: model, reasoning_effort: reasoning_effort)
           entry = history_entry
           entry[:model] = model if model
           entry[:reasoning_effort] = reasoning_effort if reasoning_effort
@@ -22,19 +22,19 @@ module RubyLLM
         end
 
         def eval_history(path: nil, model: nil, reasoning_effort: nil)
-          EvalHistory.load(path || storage_path(Report::HISTORY_DIR, "jsonl", model: model,
-                                                                              reasoning_effort: reasoning_effort))
+          EvalHistory.load(path || storage_path(Report::HISTORY_DIR, Report::HISTORY_EXT,
+                                                model: model, reasoning_effort: reasoning_effort))
         end
 
         def save_baseline!(path: nil, model: nil, reasoning_effort: nil)
-          file = path || storage_path(Report::BASELINE_DIR, "json", model: model, reasoning_effort: reasoning_effort)
+          file = path || storage_path(Report::BASELINE_DIR, Report::BASELINE_EXT, model: model, reasoning_effort: reasoning_effort)
           FileUtils.mkdir_p(File.dirname(file))
           File.write(file, JSON.pretty_generate(serialize_for_baseline))
           file
         end
 
         def compare_with_baseline(path: nil, model: nil, reasoning_effort: nil)
-          file = path || storage_path(Report::BASELINE_DIR, "json", model: model, reasoning_effort: reasoning_effort)
+          file = path || storage_path(Report::BASELINE_DIR, Report::BASELINE_EXT, model: model, reasoning_effort: reasoning_effort)
           raise ArgumentError, "No baseline found at #{file}" unless File.exist?(file)
 
           baseline_data = JSON.parse(File.read(file), symbolize_names: true)
@@ -47,13 +47,15 @@ module RubyLLM
         end
 
         def baseline_exists?(path: nil, model: nil, reasoning_effort: nil)
-          File.exist?(path || storage_path(Report::BASELINE_DIR, "json", model: model, reasoning_effort: reasoning_effort))
+          File.exist?(path || storage_path(Report::BASELINE_DIR, Report::BASELINE_EXT, model: model, reasoning_effort: reasoning_effort))
         end
 
         private
 
+        # cost_unknown is written only when true, so older files and runs that
+        # priced every case keep the same shape.
         def history_entry
-          {
+          entry = {
             date: Time.now.strftime("%Y-%m-%d"),
             score: @stats.score,
             total_cost: @stats.total_cost,
@@ -61,6 +63,8 @@ module RubyLLM
             pass_rate_ratio: @stats.pass_rate_ratio,
             cases_count: @stats.evaluated_results_count
           }
+          entry[:cost_unknown] = true if @stats.unknown_cost_results.any?
+          entry
         end
 
         def serialize_for_baseline
@@ -74,13 +78,15 @@ module RubyLLM
         end
 
         def serialize_case(result)
-          {
+          serialized = {
             name: result.name,
             passed: result.passed?,
             score: result.score,
             details: result.details,
             cost: result.cost
           }
+          serialized[:cost_unknown] = true if result.cost_unknown?
+          serialized
         end
 
         def storage_path(root_dir, extension, model:, reasoning_effort: nil)

data/lib/ruby_llm/contract/eval/unknown_cost_gate.rb

@@ -8,14 +8,6 @@ module RubyLLM
       # gate that only compares totals passes whatever those cases really cost.
       # Shared by the rake task, the pass_eval matcher and assert_eval_passes.
       module UnknownCostGate
-        MODES = %i[refuse warn].freeze
-
-        def self.validate!(mode)
-          return mode if MODES.include?(mode)
-
-          raise ArgumentError, "on_unknown_pricing must be :refuse or :warn, got #{mode.inspect}"
-        end
-
         # Returns the failure message under :refuse, nil when the gate holds.
         def self.check(reports, mode:)
           names = reports.flat_map(&:unknown_cost_results).map(&:name).uniq

data/lib/ruby_llm/contract/minitest.rb

@@ -31,8 +31,8 @@ module RubyLLM
       end
 
       def assert_eval_passes(step, eval_name, minimum_score: nil, maximum_cost: nil, context: {}, msg: nil,
-                             on_unknown_pricing: :refuse)
-        Eval::UnknownCostGate.validate!(on_unknown_pricing)
+                             on_unknown_pricing: UnknownPolicy::DEFAULT)
+        UnknownPolicy.validate!("on_unknown_pricing", on_unknown_pricing)
         report = step.run_eval(eval_name, context: context)
 
         if minimum_score

data/lib/ruby_llm/contract/pipeline/runner.rb

@@ -83,6 +83,7 @@ module RubyLLM
             total_usage: aggregate_usage(traces),
             step_traces: traces
           )
+          warn_incomplete_budget_usage(trace)
 
           Result.new(
             status: execution.status, step_results: execution.step_results,
@@ -91,6 +92,13 @@ module RubyLLM
           )
         end
 
+        def warn_incomplete_budget_usage(trace)
+          return unless @token_budget && !trace.usage_complete?
+
+          warn "[ruby_llm-contract] token_budget checked against incomplete token counts: " \
+               "a provider left out usage for at least one step, so the total undercounts"
+        end
+
         def elapsed_ms(start_time)
           ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
         end

data/lib/ruby_llm/contract/pipeline/trace.rb

@@ -40,6 +40,16 @@ module RubyLLM
           @step_traces.any? { |step_trace| step_trace.respond_to?(:cost_unknown?) && step_trace.cost_unknown? }
         end
 
+        # False when any step's provider left out token counts, so total_usage
+        # (and a token_budget checked against it) undercounts.
+        def usage_complete?
+          return true unless @step_traces.is_a?(Array)
+
+          @step_traces.none? do |step_trace|
+            step_trace.respond_to?(:usage_complete) && step_trace.usage_complete == false
+          end
+        end
+
         def to_h
           { trace_id: @trace_id, total_latency_ms: @total_latency_ms,
             total_usage: @total_usage, step_traces: @step_traces,

data/lib/ruby_llm/contract/railtie.rb

@@ -6,7 +6,7 @@ module RubyLLM
       # Ignore eval/ subdirs BEFORE Zeitwerk setup — eval files don't define
       # constants, they call define_eval on existing Step classes.
       initializer "ruby_llm_contract.ignore_eval_dirs", before: :set_autoload_paths do |app|
-        %w[app/contracts/eval app/steps/eval].each do |path|
+        RubyLLM::Contract::RAILS_EVAL_DIRS.each do |path|
           full = app.root.join(path)
           next unless full.exist?
 

data/lib/ruby_llm/contract/rake_task/suite_gate.rb

@@ -18,7 +18,7 @@ module RubyLLM
         end
 
         def self.evaluate(host_reports:, minimum_score:, maximum_cost:, fail_on_regression:,
-                          on_unknown_pricing: :refuse)
+                          on_unknown_pricing: UnknownPolicy::DEFAULT)
           new(host_reports: host_reports,
               minimum_score: minimum_score,
               maximum_cost: maximum_cost,
@@ -29,12 +29,12 @@ module RubyLLM
         attr_reader :verdict
 
         def initialize(host_reports:, minimum_score:, maximum_cost:, fail_on_regression:,
-                       on_unknown_pricing: :refuse)
+                       on_unknown_pricing: UnknownPolicy::DEFAULT)
           @host_reports = host_reports
           @minimum_score = minimum_score
           @maximum_cost = maximum_cost
           @fail_on_regression = fail_on_regression
-          @on_unknown_pricing = Eval::UnknownCostGate.validate!(on_unknown_pricing)
+          @on_unknown_pricing = UnknownPolicy.validate!("on_unknown_pricing", on_unknown_pricing)
           @verdict = build_verdict
         end
 
@@ -56,7 +56,7 @@ module RubyLLM
           passed_reports, all_passed = score_each_report
           Verdict.new(
             passed: all_passed,
-            abort_reason: all_passed ? nil : "Eval suite FAILED",
+            abort_reason: all_passed ? nil : "one or more evals did not pass",
             passed_reports: passed_reports,
             suite_cost: suite_cost
           )

data/lib/ruby_llm/contract/rake_task.rb

@@ -2,7 +2,7 @@
 
 require "rake"
 require "rake/tasklib"
-require_relative "eval/unknown_cost_gate"
+require_relative "unknown_policy"
 require_relative "rake_task/suite_gate"
 
 module RubyLLM
@@ -21,12 +21,12 @@ module RubyLLM
         @maximum_cost = nil  # nil = no cost limit; float = budget cap (suite-level)
         @eval_dirs = []      # directories to load eval files from (non-Rails)
         # With maximum_cost: :refuse fails on cases the budget cannot price; :warn gates priced cases only
-        @on_unknown_pricing = :refuse
+        @on_unknown_pricing = UnknownPolicy::DEFAULT
         @save_baseline = false
         @fail_on_regression = false
         @track_history = false
         block&.call(self)
-        Eval::UnknownCostGate.validate!(@on_unknown_pricing)
+        UnknownPolicy.validate!("on_unknown_pricing", @on_unknown_pricing)
         define_task
       end
 
@@ -134,7 +134,7 @@ module RubyLLM
           abort("STEP is required, e.g. STEP=MatchProblemsToPages") if step_name.empty?
           raw_candidates = ENV["CANDIDATES"].to_s.strip
           abort("CANDIDATES is required, e.g. CANDIDATES=gpt-5-nano,gpt-5-mini@low,gpt-5-mini") if raw_candidates.empty?
-          min_score = ENV.fetch("MIN_SCORE", "0.95").to_f
+          min_score = ENV.fetch("MIN_SCORE") { Eval::DEFAULT_MIN_SCORE }.to_f
           runs = parse_runs(ENV.fetch("RUNS", "1"))
 
           host = RubyLLM::Contract.eval_hosts.find { |h| h.name == step_name }

data/lib/ruby_llm/contract/rspec/pass_eval.rb

@@ -65,7 +65,7 @@ RSpec::Matchers.define :pass_eval do |eval_name|
   end
 
   chain :on_unknown_pricing do |mode|
-    @on_unknown_pricing = RubyLLM::Contract::Eval::UnknownCostGate.validate!(mode)
+    @on_unknown_pricing = RubyLLM::Contract::UnknownPolicy.validate!("on_unknown_pricing", mode)
   end
 
   chain :without_regressions do
@@ -82,7 +82,7 @@ RSpec::Matchers.define :pass_eval do |eval_name|
     @context ||= {}
     @minimum_score ||= nil
     @maximum_cost ||= nil
-    @on_unknown_pricing ||= :refuse
+    @on_unknown_pricing ||= RubyLLM::Contract::UnknownPolicy::DEFAULT
     @unknown_cost_failure = nil
     @unknown_cost_only = false
     @check_regressions ||= false

data/lib/ruby_llm/contract/step/base.rb

@@ -26,9 +26,9 @@ module RubyLLM
                              context: context).results.first
           end
 
-          def estimate_cost(input:, model: nil, attachment: nil)
+          def estimate_cost(input:, model: nil, attachment: nil, provider: nil)
             model_name = estimated_model_name(model)
-            model_info = CostCalculator.find_model(model_name)
+            model_info = CostCalculator.find_model(model_name, provider: provider)
             return nil unless model_info
 
             text_tokens = TokenEstimator.estimate(build_messages(input))
@@ -48,12 +48,13 @@ module RubyLLM
               output_tokens_estimate: output_tokens,
               estimated_cost: CostCalculator.calculate(
                 model_name: model_name,
-                usage: { input_tokens: input_tokens, output_tokens: output_tokens }
+                usage: { input_tokens: input_tokens, output_tokens: output_tokens },
+                provider: provider
               )
             }
           end
 
-          def estimate_eval_cost(eval_name, models: nil)
+          def estimate_eval_cost(eval_name, models: nil, provider: nil)
             defn = send(:all_eval_definitions)[eval_name.to_s]
             raise ArgumentError, "No eval '#{eval_name}' defined" unless defn
 
@@ -61,7 +62,7 @@ module RubyLLM
             cases = defn.build_dataset.cases
 
             model_list.each_with_object({}) do |model_name, result|
-              result[model_name] = estimate_eval_cost_for_model(cases, model_name)
+              result[model_name] = estimate_eval_cost_for_model(cases, model_name, provider)
             end
           end
 
@@ -90,8 +91,10 @@ module RubyLLM
             ).call
           end
 
-          KNOWN_CONTEXT_KEYS = %i[adapter model temperature max_tokens provider assume_model_exists
-                                  reasoning_effort retry_policy_override attachment].freeze
+          # Forwarded to the adapter as-is. A key known but not forwarded would be
+          # silently dropped, so the known list is built from this one.
+          ADAPTER_CONTEXT_KEYS = %i[provider assume_model_exists max_tokens reasoning_effort attachment].freeze
+          KNOWN_CONTEXT_KEYS = (%i[adapter model temperature retry_policy_override] + ADAPTER_CONTEXT_KEYS).freeze
 
           include Concerns::ContextHelpers
 
@@ -133,7 +136,7 @@ module RubyLLM
             estimate = attachment_token_estimate if respond_to?(:attachment_token_estimate)
             return [estimate, false] unless estimate.nil?
 
-            mode = respond_to?(:on_unknown_attachment_size) ? on_unknown_attachment_size : :refuse
+            mode = respond_to?(:on_unknown_attachment_size) ? on_unknown_attachment_size : UnknownPolicy::DEFAULT
             if mode == :warn
               warn "[ruby_llm-contract] attachment present but attachment_token_estimate not " \
                    "declared on #{name || self} — estimate_cost proceeds without attachment cost"
@@ -143,9 +146,9 @@ module RubyLLM
             [0, true]
           end
 
-          def estimate_eval_cost_for_model(cases, model_name)
+          def estimate_eval_cost_for_model(cases, model_name, provider)
             cases.sum do |test_case|
-              estimate = estimate_cost(input: test_case.input, model: model_name)
+              estimate = estimate_cost(input: test_case.input, model: model_name, provider: provider)
               # Two misses floor to 0.0, not one: model absent from the registry
               # (nil estimate), or present with unreadable pricing (hash whose
               # estimated_cost is nil). Documented as a floor, not a fail-closed.
@@ -193,7 +196,7 @@ module RubyLLM
 
           def runtime_settings(context)
             policy = context.key?(:retry_policy_override) ? context[:retry_policy_override] : retry_policy
-            extra = context.slice(:provider, :assume_model_exists, :max_tokens, :reasoning_effort, :attachment)
+            extra = context.slice(*ADAPTER_CONTEXT_KEYS)
 
             # Always pass the class-level `thinking` config to the adapter when
             # set, so fields like `budget` survive a per-call `reasoning_effort`
@@ -285,12 +288,18 @@ module RubyLLM
                   "model=#{trace.model} status=#{result.status} " \
                   "latency=#{trace.latency_ms}ms " \
                   "tokens=#{trace.usage&.dig(:input_tokens) || 0}+#{trace.usage&.dig(:output_tokens) || 0} " \
-                  "cost=$#{format("%.6f", trace.cost || 0)}"
+                  "cost=#{log_cost(trace)}"
             logger.info(msg)
 
             log_failed_observations(result, logger)
           end
 
+          def log_cost(trace)
+            return "unknown" if trace.respond_to?(:cost_unknown?) && trace.cost_unknown?
+
+            "$#{format("%.6f", trace.cost || 0)}"
+          end
+
           def log_failed_observations(result, logger)
             failed = result.observations.select { |o| !o[:passed] }
             return if failed.empty?

data/lib/ruby_llm/contract/step/dsl.rb

@@ -151,12 +151,10 @@ module RubyLLM
           if amount
             validate_positive!("max_cost", amount)
 
-            if on_unknown_pricing && !%i[refuse warn].include?(on_unknown_pricing)
-              raise ArgumentError, "on_unknown_pricing must be :refuse or :warn, got #{on_unknown_pricing.inspect}"
-            end
+            UnknownPolicy.validate!("on_unknown_pricing", on_unknown_pricing) if on_unknown_pricing
 
             @max_cost = amount
-            @on_unknown_pricing = on_unknown_pricing || :refuse
+            @on_unknown_pricing = on_unknown_pricing || UnknownPolicy::DEFAULT
             return @max_cost
           end
 
@@ -164,7 +162,7 @@ module RubyLLM
         end
 
         def on_unknown_pricing
-          inherited_value(:on_unknown_pricing) || :refuse
+          inherited_value(:on_unknown_pricing) || UnknownPolicy::DEFAULT
         end
 
         def attachment_token_estimate(n = nil)
@@ -182,16 +180,9 @@ module RubyLLM
         end
 
         def on_unknown_attachment_size(mode = nil)
-          if mode
-            unless %i[refuse warn].include?(mode)
-              raise ArgumentError,
-                    "on_unknown_attachment_size must be :refuse or :warn, got #{mode.inspect}"
-            end
-
-            return @on_unknown_attachment_size = mode
-          end
+          return @on_unknown_attachment_size = UnknownPolicy.validate!("on_unknown_attachment_size", mode) if mode
 
-          inherited_value(:on_unknown_attachment_size) || :refuse
+          inherited_value(:on_unknown_attachment_size) || UnknownPolicy::DEFAULT
         end
 
         def model(name = nil)

data/lib/ruby_llm/contract/step/limit_checker.rb

@@ -70,7 +70,8 @@ module RubyLLM
           estimated_output = effective_max_output || (estimated * DEFAULT_OUTPUT_RATIO)
           estimated_cost = CostCalculator.calculate(
             model_name: model_name,
-            usage: { input_tokens: estimated, output_tokens: estimated_output }
+            usage: { input_tokens: estimated, output_tokens: estimated_output },
+            provider: pricing_provider
           )
 
           if estimated_cost.nil?

data/lib/ruby_llm/contract/step/result_builder.rb

@@ -4,12 +4,21 @@ module RubyLLM
   module Contract
     module Step
       class ResultBuilder
-        def initialize(contract_definition:, output_type:, output_schema:, model:, observers:)
+        # Why the provider stopped, worded for a failed result. Anthropic also
+        # reports an exhausted context window as :max_tokens, so the message
+        # does not claim max_output was the limit hit.
+        FINISH_REASON_ERRORS = {
+          max_tokens: "provider reported token-limit termination",
+          content_filter: "provider reported content filtering or refusal"
+        }.freeze
+
+        def initialize(contract_definition:, output_type:, output_schema:, model:, observers:, max_output: nil)
           @contract_definition = contract_definition
           @output_type = output_type
           @output_schema = output_schema
           @model = model
           @observers = observers
+          @max_output = max_output
         end
 
         def error_result(error_result:, messages:)
@@ -25,13 +34,14 @@ module RubyLLM
         def success_result(response:, messages:, latency_ms:, input:)
           raw_output = response.content
           validation_result = validate_output(raw_output, input)
-          trace = Trace.new(messages: messages, model: @model, latency_ms: latency_ms, usage: response.usage)
+          trace = Trace.new(messages: messages, model: @model, latency_ms: latency_ms, usage: response.usage,
+                            **response_accounting(response))
 
           Result.new(
             status: validation_result[:status],
             raw_output: raw_output,
             parsed_output: validation_result[:parsed_output],
-            validation_errors: validation_result[:errors],
+            validation_errors: validation_errors(validation_result, response),
             trace: trace,
             observations: observations_for(validation_result, input)
           )
@@ -39,6 +49,25 @@ module RubyLLM
 
         private
 
+        # Only a failed result says why the provider stopped; a result that
+        # passed validation keeps finish_reason in its trace alone.
+        def validation_errors(validation_result, response)
+          errors = validation_result[:errors]
+          return errors if validation_result[:status] == :ok
+
+          reason = response.respond_to?(:finish_reason) ? response.finish_reason : nil
+          message = FINISH_REASON_ERRORS[reason]
+          return errors unless message
+
+          message += "; configured max_output: #{@max_output}" if reason == :max_tokens && @max_output
+          [*errors, message]
+        end
+
+        # Custom adapters may return any object with `content` and `usage`.
+        def response_accounting(response)
+          response.respond_to?(:trace_accounting) ? response.trace_accounting : {}
+        end
+
         def observations_for(validation_result, input)
           return [] unless validation_result[:status] == :ok && @observers.any?