ruby_llm-contract 0.10.6 → 1.1.0

data/lib/ruby_llm/contract/eval/retry_optimizer.rb

@@ -13,6 +13,10 @@ module RubyLLM
       #   result.print_summary
       #   result.to_dsl  # => copy-paste retry_policy
       class RetryOptimizer
+        # Documented CANDIDATES= shorthand, parsed by
+        # OptimizeRakeTask#parse_candidates and rendered in this table's headers.
+        EFFORT_SEPARATOR = "@"
+
         Result = Struct.new(:step_name, :eval_names, :candidate_labels, :score_matrix,
                             :constraining_eval, :chain, :chain_details, keyword_init: true) do
           # Terminology alias — `hardest_eval` is the narrative name used in docs;
@@ -80,11 +84,10 @@ module RubyLLM
           end
 
           def short_candidate_label(label)
-            label
-              .sub("gpt-5-", "")
-              .sub("gpt-4.1", "4.1")
-              .sub(" (effort: ", "@")
-              .sub(")", "")
+            config = CandidateLabel.parse(label)
+            model = config[:model].sub("gpt-5-", "").sub("gpt-4.1", "4.1")
+            effort = config[:reasoning_effort]
+            effort ? "#{model}#{EFFORT_SEPARATOR}#{effort}" : model
           end
 
           def print_dsl(io)
@@ -176,11 +179,9 @@ module RubyLLM
         def build_chain(matrix, labels, evals)
           total = evals.size
 
-          # Find cheapest model that passes every eval — the safe fallback.
           safe_fallback = labels.find { |l| evals.all? { |e| (matrix.dig(e, l) || 0) >= @min_score } }
           return [[], []] unless safe_fallback
 
-          # Prepend cheaper models that pass a strict subset.
           chain = []
           details = []
           covered_evals = Set.new
@@ -193,27 +194,16 @@ module RubyLLM
             next if new_additions.empty?
 
             covered_evals.merge(new_additions)
-            chain << parse_label_to_config(label)
+            chain << CandidateLabel.parse(label)
             details << { label: label, passes: new_additions.size, cost: label }
           end
 
-          # Always end with the safe fallback.
-          chain << parse_label_to_config(safe_fallback)
+          chain << CandidateLabel.parse(safe_fallback)
           details << { label: safe_fallback, passes: total, cost: safe_fallback }
 
           [chain, details]
         end
 
-        def parse_label_to_config(label)
-          if label.match?(/\(effort: (\w+)\)/)
-            model = label.sub(/\s*\(effort:.*/, "").strip
-            effort = label.match(/\(effort: (\w+)\)/)[1]
-            { model: model, reasoning_effort: effort }
-          else
-            { model: label }
-          end
-        end
-
         def empty_result(evals)
           Result.new(
             step_name: @step.name || @step.to_s,

data/lib/ruby_llm/contract/eval/step_expectation_applier.rb

@@ -58,7 +58,8 @@ module RubyLLM
             label: "FAIL",
             details: "step expectations failed: #{failure_details}",
             duration_ms: result.duration_ms,
-            cost: result.cost
+            cost: result.cost,
+            cost_unknown: result.cost_unknown?
           )
         end
       end

data/lib/ruby_llm/contract/eval/trait_evaluator.rb

@@ -3,8 +3,6 @@
 module RubyLLM
   module Contract
     module Eval
-      # Extracted from Runner to reduce class length.
-      # Evaluates expected_traits against parsed output.
       module TraitEvaluator
         private
 

data/lib/ruby_llm/contract/eval/unknown_cost_gate.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Eval
+      # Suite-level counterpart of the step-level `max_cost ... on_unknown_pricing:`.
+      # A report's total_cost counts an unpriced case as 0.0, so a maximum_cost
+      # gate that only compares totals passes whatever those cases really cost.
+      # Shared by the rake task, the pass_eval matcher and assert_eval_passes.
+      module UnknownCostGate
+        MODES = %i[refuse warn].freeze
+
+        def self.validate!(mode)
+          return mode if MODES.include?(mode)
+
+          raise ArgumentError, "on_unknown_pricing must be :refuse or :warn, got #{mode.inspect}"
+        end
+
+        # Returns the failure message under :refuse, nil when the gate holds.
+        def self.check(reports, mode:)
+          names = reports.flat_map(&:unknown_cost_results).map(&:name).uniq
+          return nil if names.empty?
+
+          if mode == :warn
+            warn "[ruby_llm-contract] #{describe(names)} - maximum_cost checked against priced cases only"
+            return nil
+          end
+
+          "#{describe(names)}. Register pricing via CostCalculator.register_model " \
+            "or set on_unknown_pricing to :warn to check maximum_cost against priced cases only."
+        end
+
+        def self.describe(names)
+          "#{names.length} case(s) have unknown cost (model has no pricing data): #{names.join(", ")}"
+        end
+        private_class_method :describe
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval.rb

@@ -22,7 +22,9 @@ require_relative "eval/report_presenter"
 require_relative "eval/report_storage"
 require_relative "eval/report"
 require_relative "eval/aggregated_report"
+require_relative "eval/unknown_cost_gate"
 require_relative "eval/eval_definition"
+require_relative "eval/candidate_label"
 require_relative "eval/model_comparison"
 require_relative "eval/baseline_diff"
 require_relative "eval/prompt_diff_serializer"

data/lib/ruby_llm/contract/minitest.rb

@@ -30,7 +30,9 @@ module RubyLLM
         refute result.ok?, msg || "Expected step result NOT to satisfy contract, but it passed"
       end
 
-      def assert_eval_passes(step, eval_name, minimum_score: nil, maximum_cost: nil, context: {}, msg: nil)
+      def assert_eval_passes(step, eval_name, minimum_score: nil, maximum_cost: nil, context: {}, msg: nil,
+                             on_unknown_pricing: :refuse)
+        Eval::UnknownCostGate.validate!(on_unknown_pricing)
         report = step.run_eval(eval_name, context: context)
 
         if minimum_score
@@ -42,6 +44,9 @@ module RubyLLM
         end
 
         if maximum_cost
+          unknown_cost_failure = Eval::UnknownCostGate.check([report], mode: on_unknown_pricing)
+          assert unknown_cost_failure.nil?,
+                 msg || "Expected #{eval_name} eval to stay within maximum cost, but #{unknown_cost_failure}"
           assert report.total_cost <= maximum_cost,
                  msg || "Expected #{eval_name} eval cost <= $#{format("%.4f", maximum_cost)}, got $#{format("%.4f", report.total_cost)}"
         end

data/lib/ruby_llm/contract/pipeline/runner.rb

@@ -95,7 +95,6 @@ module RubyLLM
           ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
         end
 
-        # Encapsulates mutable state during pipeline execution
         class ExecutionState
           attr_reader :trace_id, :step_results, :step_traces, :outputs_by_step,
                       :current_input, :status, :failed_step

data/lib/ruby_llm/contract/pipeline/trace.rb

@@ -33,6 +33,13 @@ module RubyLLM
           value.dig(*rest)
         end
 
+        # total_cost sums only the steps that could be priced.
+        def cost_unknown?
+          return false unless @step_traces.is_a?(Array)
+
+          @step_traces.any? { |step_trace| step_trace.respond_to?(:cost_unknown?) && step_trace.cost_unknown? }
+        end
+
         def to_h
           { trace_id: @trace_id, total_latency_ms: @total_latency_ms,
             total_usage: @total_usage, step_traces: @step_traces,

data/lib/ruby_llm/contract/rake_task/suite_gate.rb

@@ -3,21 +3,10 @@
 module RubyLLM
   module Contract
     class RakeTask < ::Rake::TaskLib
-      # Encapsulates the pass/fail gate that runs after `RakeTask#define_task`
-      # has collected eval reports. Extracted from the prior `define_task`
-      # god-method so each gating dimension (cost, score, regression) is
-      # testable in isolation.
-      #
-      # Returns a `Verdict` value object with:
-      #   - `passed?`         — overall gate verdict
-      #   - `abort_reason`    — String for `abort` when `passed? == false`, nil otherwise
-      #   - `passed_reports`  — [[host, report], ...] of reports that individually passed
-      #                         (used to decide which baselines to save)
-      #   - `suite_cost`      — total cost across all reports
-      #
       # Gate ordering (preserved from pre-refactor behaviour):
-      #   1. cost gate runs FIRST — if `maximum_cost` set and exceeded, the
-      #      suite aborts before any score check; passed_reports is empty.
+      #   1. cost gate runs FIRST — if `maximum_cost` set and exceeded, or any
+      #      case has unknown cost under on_unknown_pricing :refuse, the suite
+      #      aborts before any score check; passed_reports is empty.
       #   2. score gate runs per-report; a report passes if
       #      `report_meets_score?` AND `!check_regression`.
       #   3. overall passed = ALL reports passed AND cost gate not tripped.
@@ -28,20 +17,24 @@ module RubyLLM
           end
         end
 
-        def self.evaluate(host_reports:, minimum_score:, maximum_cost:, fail_on_regression:)
+        def self.evaluate(host_reports:, minimum_score:, maximum_cost:, fail_on_regression:,
+                          on_unknown_pricing: :refuse)
           new(host_reports: host_reports,
               minimum_score: minimum_score,
               maximum_cost: maximum_cost,
-              fail_on_regression: fail_on_regression).verdict
+              fail_on_regression: fail_on_regression,
+              on_unknown_pricing: on_unknown_pricing).verdict
         end
 
         attr_reader :verdict
 
-        def initialize(host_reports:, minimum_score:, maximum_cost:, fail_on_regression:)
+        def initialize(host_reports:, minimum_score:, maximum_cost:, fail_on_regression:,
+                       on_unknown_pricing: :refuse)
           @host_reports = host_reports
           @minimum_score = minimum_score
           @maximum_cost = maximum_cost
           @fail_on_regression = fail_on_regression
+          @on_unknown_pricing = Eval::UnknownCostGate.validate!(on_unknown_pricing)
           @verdict = build_verdict
         end
 
@@ -49,11 +42,12 @@ module RubyLLM
 
         def build_verdict
           suite_cost = compute_suite_cost
+          cost_failure = cost_failure_message(suite_cost)
 
-          if cost_exceeded?(suite_cost)
+          if cost_failure
             return Verdict.new(
               passed: false,
-              abort_reason: cost_abort_message(suite_cost),
+              abort_reason: cost_failure,
               passed_reports: [],
               suite_cost: suite_cost
             )
@@ -72,8 +66,11 @@ module RubyLLM
           @host_reports.sum { |_host, report| report.total_cost }
         end
 
-        def cost_exceeded?(suite_cost)
-          @maximum_cost && suite_cost > @maximum_cost
+        def cost_failure_message(suite_cost)
+          return nil unless @maximum_cost
+          return cost_abort_message(suite_cost) if suite_cost > @maximum_cost
+
+          Eval::UnknownCostGate.check(@host_reports.map(&:last), mode: @on_unknown_pricing)
         end
 
         def cost_abort_message(suite_cost)

data/lib/ruby_llm/contract/rake_task.rb

@@ -2,13 +2,15 @@
 
 require "rake"
 require "rake/tasklib"
+require_relative "eval/unknown_cost_gate"
 require_relative "rake_task/suite_gate"
 
 module RubyLLM
   module Contract
     class RakeTask < ::Rake::TaskLib
       attr_accessor :name, :context, :fail_on_empty, :minimum_score, :maximum_cost,
-                    :eval_dirs, :save_baseline, :fail_on_regression, :track_history
+                    :eval_dirs, :save_baseline, :fail_on_regression, :track_history,
+                    :on_unknown_pricing
 
       def initialize(name = :"ruby_llm_contract:eval", &block)
         super()
@@ -18,10 +20,13 @@ module RubyLLM
         @minimum_score = nil # nil = require 100%; float = threshold
         @maximum_cost = nil  # nil = no cost limit; float = budget cap (suite-level)
         @eval_dirs = []      # directories to load eval files from (non-Rails)
+        # With maximum_cost: :refuse fails on cases the budget cannot price; :warn gates priced cases only
+        @on_unknown_pricing = :refuse
         @save_baseline = false
         @fail_on_regression = false
         @track_history = false
         block&.call(self)
+        Eval::UnknownCostGate.validate!(@on_unknown_pricing)
         define_task
       end
 
@@ -44,7 +49,8 @@ module RubyLLM
             host_reports: host_reports,
             minimum_score: @minimum_score,
             maximum_cost: @maximum_cost,
-            fail_on_regression: @fail_on_regression
+            fail_on_regression: @fail_on_regression,
+            on_unknown_pricing: @on_unknown_pricing
           )
 
           abort "\nEval suite FAILED: #{verdict.abort_reason}" unless verdict.passed?
@@ -164,7 +170,7 @@ module RubyLLM
                     Array(JSON.parse(raw))
                   else
                     raw.split(",").map(&:strip).reject(&:empty?).map do |entry|
-                      model, effort = entry.split("@", 2)
+                      model, effort = entry.split(Eval::RetryOptimizer::EFFORT_SEPARATOR, 2)
                       config = { model: model.strip }
                       config[:reasoning_effort] = effort.strip if effort && !effort.empty?
                       config
@@ -191,7 +197,6 @@ module RubyLLM
       end
     end
 
-    # Auto-register the optimize task when this file is loaded
     OptimizeRakeTask.new
   end
 end

data/lib/ruby_llm/contract/rspec/pass_eval.rb

@@ -64,6 +64,10 @@ RSpec::Matchers.define :pass_eval do |eval_name|
     @maximum_cost = cost
   end
 
+  chain :on_unknown_pricing do |mode|
+    @on_unknown_pricing = RubyLLM::Contract::Eval::UnknownCostGate.validate!(mode)
+  end
+
   chain :without_regressions do
     @check_regressions = true
   end
@@ -78,6 +82,9 @@ RSpec::Matchers.define :pass_eval do |eval_name|
     @context ||= {}
     @minimum_score ||= nil
     @maximum_cost ||= nil
+    @on_unknown_pricing ||= :refuse
+    @unknown_cost_failure = nil
+    @unknown_cost_only = false
     @check_regressions ||= false
     @comparison_step ||= nil
     @error = nil
@@ -97,7 +104,11 @@ RSpec::Matchers.define :pass_eval do |eval_name|
                  @report.passed?
                end
 
-    cost_ok = @maximum_cost ? @report.total_cost <= @maximum_cost : true
+    cost_ok = true
+    if @maximum_cost
+      @unknown_cost_failure = RubyLLM::Contract::Eval::UnknownCostGate.check([@report], mode: @on_unknown_pricing)
+      cost_ok = @report.total_cost <= @maximum_cost && @unknown_cost_failure.nil?
+    end
 
     regression_ok = if @prompt_diff
                       @prompt_diff.safe_to_switch?
@@ -108,6 +119,7 @@ RSpec::Matchers.define :pass_eval do |eval_name|
                       true
                     end
 
+    @unknown_cost_only = score_ok && regression_ok && @report.total_cost <= @maximum_cost if @unknown_cost_failure
     score_ok && cost_ok && regression_ok
   rescue StandardError => e
     @error = e
@@ -115,71 +127,87 @@ RSpec::Matchers.define :pass_eval do |eval_name|
   end
 
   failure_message do
-    if @prompt_diff && !@prompt_diff.safe_to_switch?
-      msg = "expected #{@eval_name} eval to be safe to switch from baseline prompt\n"
-
-      # Check empty sides first — most fundamental problem
-      bl_empty = @prompt_diff.baseline_empty?
-      cd_empty = @prompt_diff.candidate_empty?
-      if bl_empty || cd_empty
-        msg += "  One side has no evaluated cases (all skipped or no adapter?)\n"
-        if sample_response_only_compare?
-          msg += "  compare_with ignores sample_response; pass model: or with_context(adapter: ...)\n"
+    other_failure_message = lambda do
+      if @prompt_diff && !@prompt_diff.safe_to_switch?
+        msg = "expected #{@eval_name} eval to be safe to switch from baseline prompt\n"
+
+        # Check empty sides first — most fundamental problem
+        bl_empty = @prompt_diff.baseline_empty?
+        cd_empty = @prompt_diff.candidate_empty?
+        if bl_empty || cd_empty
+          msg += "  One side has no evaluated cases (all skipped or no adapter?)\n"
+          if sample_response_only_compare?
+            msg += "  compare_with ignores sample_response; pass model: or with_context(adapter: ...)\n"
+          end
+          msg += "  Candidate score: #{@prompt_diff.candidate_score}, Baseline score: #{@prompt_diff.baseline_score}"
+          next msg
         end
-        msg += "  Candidate score: #{@prompt_diff.candidate_score}, Baseline score: #{@prompt_diff.baseline_score}"
-        next msg
-      end
 
-      # Check dataset comparability — names, inputs, AND expected must match
-      unless @prompt_diff.cases_comparable?
-        unless @prompt_diff.case_names_match?
-          mm = @prompt_diff.mismatched_cases
-          msg += "  Case set mismatch — candidate and baseline must have identical cases:\n"
-          mm[:only_in_baseline].each { |n| msg += "    only in baseline: #{n}\n" }
-          mm[:only_in_candidate].each { |n| msg += "    only in candidate: #{n}\n" }
-        end
-        @prompt_diff.input_mismatches.each do |m|
-          msg += "  Input mismatch for '#{m[:case]}' — same name but different inputs\n"
+        # Check dataset comparability — names, inputs, AND expected must match
+        unless @prompt_diff.cases_comparable?
+          unless @prompt_diff.case_names_match?
+            mm = @prompt_diff.mismatched_cases
+            msg += "  Case set mismatch — candidate and baseline must have identical cases:\n"
+            mm[:only_in_baseline].each { |n| msg += "    only in baseline: #{n}\n" }
+            mm[:only_in_candidate].each { |n| msg += "    only in candidate: #{n}\n" }
+          end
+          @prompt_diff.input_mismatches.each do |m|
+            msg += "  Input mismatch for '#{m[:case]}' — same name but different inputs\n"
+          end
+          @prompt_diff.expected_mismatches.each do |m|
+            msg += "  Expected mismatch for '#{m[:case]}' — same name/input but different expected values\n"
+          end
+          next msg
         end
-        @prompt_diff.expected_mismatches.each do |m|
-          msg += "  Expected mismatch for '#{m[:case]}' — same name/input but different expected values\n"
+
+        # Check per-case score regressions (even if global average is flat)
+        if @prompt_diff.score_regressions.any?
+          msg += "  Per-case score regressions (#{@prompt_diff.score_regressions.length}):\n"
+          @prompt_diff.score_regressions.each do |r|
+            msg += "    #{r[:case]}: #{r[:baseline_score]} -> #{r[:candidate_score]} (#{r[:delta]})\n"
+          end
+          msg += "  Score delta: #{@prompt_diff.score_delta}"
+          next msg
         end
-        next msg
-      end
 
-      # Check per-case score regressions (even if global average is flat)
-      if @prompt_diff.score_regressions.any?
-        msg += "  Per-case score regressions (#{@prompt_diff.score_regressions.length}):\n"
-        @prompt_diff.score_regressions.each do |r|
-          msg += "    #{r[:case]}: #{r[:baseline_score]} -> #{r[:candidate_score]} (#{r[:delta]})\n"
+        # Check pass/fail regressions and removed cases
+        removed = @prompt_diff.removed_passing_cases
+        reg_count = @prompt_diff.regressions.length + removed.length
+        msg += "  Found #{reg_count} regression(s):\n"
+        @prompt_diff.regressions.each do |r|
+          msg += "    #{r[:case]}: was PASS, now FAIL -- #{r[:detail]}\n"
+        end
+        removed.each do |name|
+          msg += "    #{name}: REMOVED (was passing in baseline)\n"
         end
         msg += "  Score delta: #{@prompt_diff.score_delta}"
         next msg
       end
 
-      # Check pass/fail regressions and removed cases
-      removed = @prompt_diff.removed_passing_cases
-      reg_count = @prompt_diff.regressions.length + removed.length
-      msg += "  Found #{reg_count} regression(s):\n"
-      @prompt_diff.regressions.each do |r|
-        msg += "    #{r[:case]}: was PASS, now FAIL -- #{r[:detail]}\n"
-      end
-      removed.each do |name|
-        msg += "    #{name}: REMOVED (was passing in baseline)\n"
+      msg = format_failure_message(@eval_name, @error, @report, @minimum_score, @maximum_cost)
+      if @diff&.regressed?
+        msg += "\n\nRegressions from baseline:\n"
+        @diff.regressions.each do |r|
+          msg += "  #{r[:case]}: was PASS, now FAIL -- #{r[:detail]}\n"
+        end
+        @diff.skipped_passing_cases.each do |s|
+          msg += "  #{s[:case]}: was PASS, now SKIPPED -- #{s[:detail]}\n"
+        end
+        msg += "  Score delta: #{@diff.score_delta}"
       end
-      msg += "  Score delta: #{@prompt_diff.score_delta}"
-      next msg
+      msg
     end
 
-    msg = format_failure_message(@eval_name, @error, @report, @minimum_score, @maximum_cost)
-    if @diff&.regressed?
-      msg += "\n\nRegressions from baseline:\n"
-      @diff.regressions.each do |r|
-        msg += "  #{r[:case]}: was PASS, now FAIL -- #{r[:detail]}\n"
-      end
-      msg += "  Score delta: #{@diff.score_delta}"
+    # Cost first, as in the rake suite gate: an unpriced total cannot be trusted.
+    # An exception outranks it, and any other failure is still listed.
+    if @unknown_cost_failure && @error.nil?
+      unknown_cost = "expected #{@eval_name} eval to stay within maximum cost, but #{@unknown_cost_failure}"
+      next unknown_cost if @unknown_cost_only
+
+      next "#{unknown_cost}\n\n#{other_failure_message.call}"
     end
-    msg
+
+    other_failure_message.call
   end
 
   failure_message_when_negated do

data/lib/ruby_llm/contract/step/base.rb

@@ -6,6 +6,11 @@ module RubyLLM
       class Base
         DEFAULT_OUTPUT_TOKENS = 256
 
+        # Eval::CaseExecutor substring-matches this to choose skip over raise.
+        # Reworded without the consumer, every skipped eval case becomes a hard
+        # failure of the whole run, so both ends read it from here.
+        NO_ADAPTER_MESSAGE = "No adapter configured"
+
         def self.inherited(subclass)
           super
           Contract.register_eval_host(subclass) if respond_to?(:eval_defined?) && eval_defined?
@@ -141,7 +146,10 @@ module RubyLLM
           def estimate_eval_cost_for_model(cases, model_name)
             cases.sum do |test_case|
               estimate = estimate_cost(input: test_case.input, model: model_name)
-              estimate ? estimate[:estimated_cost] : 0.0
+              # Two misses floor to 0.0, not one: model absent from the registry
+              # (nil estimate), or present with unreadable pricing (hash whose
+              # estimated_cost is nil). Documented as a floor, not a fail-closed.
+              (estimate && estimate[:estimated_cost]) || 0.0
             end.round(6)
           end
 
@@ -225,8 +233,9 @@ module RubyLLM
             adapter = context[:adapter] || RubyLLM::Contract.configuration.default_adapter
             return adapter if adapter
 
-            raise RubyLLM::Contract::Error, "No adapter configured. Set one with RubyLLM::Contract.configure " \
-                                            "{ |c| c.default_adapter = ... } or pass context: { adapter: ... }"
+            raise RubyLLM::Contract::Error,
+                  "#{NO_ADAPTER_MESSAGE}. Set one with RubyLLM::Contract.configure " \
+                  "{ |c| c.default_adapter = ... } or pass context: { adapter: ... }"
           end
 
           # ADR-0021 deliverable 2: narrow ArgumentError rescue to DSL-setup phase only.

data/lib/ruby_llm/contract/step/dsl.rb

@@ -3,8 +3,6 @@
 module RubyLLM
   module Contract
     module Step
-      # Extracted from Base to reduce class length.
-      # DSL accessor methods for step definition (input_type, output_type, prompt, etc.).
       module Dsl # rubocop:disable Metrics/ModuleLength
         # Sentinel signalling "explicitly reset" (`some_attr(:default)`).
         # Distinguishes reset (lookup stops at this class, returns nil) from
@@ -65,8 +63,8 @@ module RubyLLM
 
         def output_schema(&block)
           if block
-            require "ruby_llm/schema"
-            @output_schema = ::RubyLLM::Schema.create(&block)
+            require "schematist"
+            @output_schema = ::Schematist::Schema.create(&block)
           elsif defined?(@output_schema)
             @output_schema
           elsif superclass.respond_to?(:output_schema)

data/lib/ruby_llm/contract/step/limit_checker.rb

@@ -3,8 +3,6 @@
 module RubyLLM
   module Contract
     module Step
-      # Extracted from Runner to reduce class length.
-      # Handles input token limit and cost limit checks.
       module LimitChecker
         private
 

data/lib/ruby_llm/contract/step/retry_executor.rb

@@ -3,8 +3,6 @@
 module RubyLLM
   module Contract
     module Step
-      # Extracted from Base to reduce class length.
-      # Handles retry logic: run_with_retry, build_retry_result, aggregate usage, build attempt entries.
       module RetryExecutor
         include Concerns::UsageAggregator
 

data/lib/ruby_llm/contract/step/trace.rb

@@ -50,6 +50,25 @@ module RubyLLM
           )
         end
 
+        # True when a call ran on a named model, used tokens, and still got no
+        # cost, so any total built from this trace undercounts. A zero-token call
+        # (Test adapter, sample_response) costs 0 whatever the pricing. A retried
+        # step is judged per attempt: a priced subtotal must not hide an unpriced
+        # attempt, and the merged trace may have re-priced on the last model.
+        def cost_unknown?
+          if @attempts.is_a?(Array) && !@attempts.empty?
+            @attempts.any? { |attempt| self.class.unpriced?(attempt[:model], attempt[:usage], attempt[:cost]) }
+          else
+            self.class.unpriced?(@model, @usage, @cost)
+          end
+        end
+
+        def self.unpriced?(model, usage, cost)
+          return false unless model && cost.nil? && usage.is_a?(Hash)
+
+          ((usage[:input_tokens] || 0) + (usage[:output_tokens] || 0)).positive?
+        end
+
         def to_h
           { messages: @messages, model: @model, latency_ms: @latency_ms,
             usage: @usage, attempts: @attempts, cost: @cost }.compact

data/lib/ruby_llm/contract/token_estimator.rb

@@ -23,8 +23,6 @@ module RubyLLM
     module TokenEstimator
       CHARS_PER_TOKEN = 4
 
-      # Heuristic estimate. Returns an integer token count.
-      # See module docstring for accuracy caveats.
       def self.estimate(messages)
         return 0 unless messages.is_a?(Array)
 

data/lib/ruby_llm/contract/version.rb

@@ -2,6 +2,6 @@
 
 module RubyLLM
   module Contract
-    VERSION = "0.10.6"
+    VERSION = "1.1.0"
   end
 end

data/lib/ruby_llm/contract.rb

@@ -21,8 +21,6 @@ module RubyLLM
         step_adapter_overrides.clear
       end
 
-      # --- Eval host registry ---
-
       def register_eval_host(klass)
         eval_hosts << klass unless eval_hosts.include?(klass)
       end
@@ -101,9 +99,7 @@ module RubyLLM
 
       private
 
-      # Filter stale hosts, deduplicate by name (last wins), prune registry in-place
       def live_eval_hosts
-        # Remove hosts without evals
         @eval_hosts&.reject! { |h| !h.respond_to?(:eval_defined?) || !h.eval_defined? }
 
         # Deduplicate: if two classes share a name (reload), keep the latest