ruby_llm-contract 1.1.0 → 1.1.2

data/lib/ruby_llm/contract/step/retry_executor.rb

@@ -6,6 +6,9 @@ module RubyLLM
       module RetryExecutor
         include Concerns::UsageAggregator
 
+        # Copied from each attempt's trace into its attempts entry when present.
+        ATTEMPT_TRACE_FIELDS = %i[usage latency_ms cost finish_reason].freeze
+
         private
 
         def run_with_retry(input, adapter:, default_model:, policy:, context_temperature: nil, extra_options: {})
@@ -39,7 +42,9 @@ module RubyLLM
             observations: last.observations,
             trace: last.trace.merge(
               attempts: attempt_log, usage: aggregated_usage,
-              cost: total_cost, latency_ms: total_latency
+              cost: total_cost, latency_ms: total_latency,
+              usage_complete: aggregate_flag(all_attempts, :usage_complete),
+              cost_complete: aggregate_cost_complete(all_attempts)
             )
           )
         end
@@ -53,12 +58,38 @@ module RubyLLM
         end
 
         def append_trace_fields(entry, trace)
-          entry[:usage] = trace.usage if trace.respond_to?(:usage) && trace.usage
-          entry[:latency_ms] = trace.latency_ms if trace.respond_to?(:latency_ms) && trace.latency_ms
-          entry[:cost] = trace.cost if trace.respond_to?(:cost) && trace.cost
+          ATTEMPT_TRACE_FIELDS.each do |field|
+            value = trace_value(trace, field)
+            entry[field] = value if value
+          end
+          entry[:cost_unknown] = true if trace_value(trace, :cost_unknown?)
+          entry[:usage_complete] = false if trace_value(trace, :usage_complete) == false
           entry
         end
 
+        def trace_value(trace, method)
+          trace.public_send(method) if trace.respond_to?(method)
+        end
+
+        # The subtotal is complete only if every attempt was; the last attempt's
+        # flag must not speak for it. nil when no attempt reported accounting
+        # state, so a retried Test-adapter step serializes exactly as before.
+        def aggregate_cost_complete(all_attempts)
+          traces = all_attempts.map { |a| a[:result].trace }
+          return false if traces.any? { |trace| trace.respond_to?(:cost_unknown?) && trace.cost_unknown? }
+
+          aggregate_flag(all_attempts, :cost_complete)
+        end
+
+        def aggregate_flag(all_attempts, flag)
+          flags = all_attempts.map { |a| a[:result].trace }
+                              .select { |trace| trace.respond_to?(flag) }
+                              .map { |trace| trace.public_send(flag) }.compact
+          return nil if flags.empty?
+
+          flags.all?
+        end
+
         def sum_attempt_costs(all_attempts)
           costs = extract_trace_values(all_attempts, :cost)
           return nil if costs.empty?

data/lib/ruby_llm/contract/step/runner.rb

@@ -59,7 +59,8 @@ module RubyLLM
             output_type: @config.output_type,
             output_schema: @config.output_schema,
             model: @config.model,
-            observers: @config.observers
+            observers: @config.observers,
+            max_output: @config.effective_max_output
           )
         end
 
@@ -87,6 +88,11 @@ module RubyLLM
           @config.on_unknown_attachment_size
         end
 
+        # The provider the adapter will call, so the estimate uses its price.
+        def pricing_provider
+          @config.extra_options&.dig(:provider)
+        end
+
         def attachment_present?
           opts = @config.extra_options
           !opts.nil? && !opts[:attachment].nil?

data/lib/ruby_llm/contract/step/runner_config.rb

@@ -28,8 +28,8 @@ module RubyLLM
         def self.build(input_type:, output_type:, prompt_block:, contract_definition:,
                        adapter:, model:,
                        output_schema: nil, max_output: nil,
-                       max_input: nil, max_cost: nil, on_unknown_pricing: :refuse,
-                       attachment_token_estimate: nil, on_unknown_attachment_size: :refuse,
+                       max_input: nil, max_cost: nil, on_unknown_pricing: UnknownPolicy::DEFAULT,
+                       attachment_token_estimate: nil, on_unknown_attachment_size: UnknownPolicy::DEFAULT,
                        temperature: nil, extra_options: {}, observers: [])
           new(
             input_type: input_type, output_type: output_type,

data/lib/ruby_llm/contract/step/trace.rb

@@ -7,19 +7,34 @@ module RubyLLM
         include Concerns::TraceEquality
         include Concerns::DeepFreeze
 
-        attr_reader :messages, :model, :latency_ms, :usage, :attempts, :cost
-
-        def initialize(messages: nil, model: nil, latency_ms: nil, usage: nil, attempts: nil, cost: nil)
+        attr_reader :messages, :model, :latency_ms, :usage, :attempts, :cost,
+                    :usage_complete, :cost_complete, :finish_reason
+
+        # Marks `cost:` as not given, as distinct from an explicit nil. Only an
+        # omitted cost is priced from the registry; nil from an adapter that
+        # reports its own cost means "unknown" and must stay unknown.
+        COST_UNSET = Object.new.freeze
+
+        # usage_complete / cost_complete are nil for traces that carry no
+        # accounting state (Test adapter, older adapters, hand-built traces);
+        # those keep the token-count rule in `unpriced?`.
+        def initialize(messages: nil, model: nil, latency_ms: nil, usage: nil, attempts: nil, cost: COST_UNSET,
+                       usage_complete: nil, cost_complete: nil, finish_reason: nil)
           @messages = deep_dup_freeze(messages)
           @model = model.frozen? ? model : model&.dup&.freeze
           @latency_ms = latency_ms
           @usage = deep_dup_freeze(usage)
           @attempts = deep_dup_freeze(attempts)
-          @cost = cost || CostCalculator.calculate(model_name: model, usage: usage)
+          @usage_complete = usage_complete
+          @cost_complete = cost_complete
+          @finish_reason = finish_reason
+          @cost = resolve_cost(cost)
+          @keep_nil_cost = keep_nil_cost?(cost)
           freeze
         end
 
-        KNOWN_KEYS = %i[messages model latency_ms usage attempts cost].freeze
+        KNOWN_KEYS = %i[messages model latency_ms usage attempts cost
+                        usage_complete cost_complete finish_reason].freeze
 
         def [](key)
           return nil unless KNOWN_KEYS.include?(key.to_sym)
@@ -39,27 +54,27 @@ module RubyLLM
         end
         alias has_key? key?
 
+        # Never re-prices: a retry merges a subtotal and an aggregate usage that
+        # no single model's pricing describes.
         def merge(**overrides)
-          self.class.new(
-            messages: overrides.fetch(:messages, @messages),
-            model: overrides.fetch(:model, @model),
-            latency_ms: overrides.fetch(:latency_ms, @latency_ms),
-            usage: overrides.fetch(:usage, @usage),
-            attempts: overrides.fetch(:attempts, @attempts),
-            cost: overrides.fetch(:cost, @cost)
-          )
+          self.class.new(**KNOWN_KEYS.to_h { |key| [key, overrides.fetch(key) { public_send(key) }] })
         end
 
-        # True when a call ran on a named model, used tokens, and still got no
-        # cost, so any total built from this trace undercounts. A zero-token call
+        # True when a total built from this trace undercounts: the adapter said
+        # its cost does not cover the call, or (no accounting state) a call ran
+        # on a named model, used tokens, and still got no cost. A zero-token call
         # (Test adapter, sample_response) costs 0 whatever the pricing. A retried
         # step is judged per attempt: a priced subtotal must not hide an unpriced
-        # attempt, and the merged trace may have re-priced on the last model.
+        # attempt.
         def cost_unknown?
+          return true if @cost_complete == false
+
           if @attempts.is_a?(Array) && !@attempts.empty?
-            @attempts.any? { |attempt| self.class.unpriced?(attempt[:model], attempt[:usage], attempt[:cost]) }
+            @attempts.any? do |attempt|
+              attempt[:cost_unknown] || self.class.unpriced?(attempt[:model], attempt[:usage], attempt[:cost])
+            end
           else
-            self.class.unpriced?(@model, @usage, @cost)
+            @cost_complete.nil? && self.class.unpriced?(@model, @usage, @cost)
           end
         end
 
@@ -70,8 +85,11 @@ module RubyLLM
         end
 
         def to_h
-          { messages: @messages, model: @model, latency_ms: @latency_ms,
-            usage: @usage, attempts: @attempts, cost: @cost }.compact
+          hash = { messages: @messages, model: @model, latency_ms: @latency_ms,
+                   usage: @usage, attempts: @attempts, cost: @cost, usage_complete: @usage_complete,
+                   cost_complete: @cost_complete, finish_reason: @finish_reason }.compact
+          hash[:cost] = nil if @keep_nil_cost
+          hash
         end
 
         def to_s
@@ -80,6 +98,22 @@ module RubyLLM
 
         private
 
+        # A given nil cost that the registry would price must stay `cost: nil` in
+        # to_h, or `Trace.new(**to_h)` would come back priced. Otherwise it is
+        # left out, so a merged trace compares equal to the one it came from.
+        def keep_nil_cost?(cost)
+          return false unless cost.nil? && @cost_complete.nil?
+
+          !CostCalculator.calculate(model_name: @model, usage: @usage).nil?
+        end
+
+        def resolve_cost(cost)
+          return cost unless cost.equal?(COST_UNSET)
+          return nil unless @cost_complete.nil?
+
+          CostCalculator.calculate(model_name: @model, usage: @usage)
+        end
+
         def build_summary_parts
           parts = [@model || "no-model"]
           parts << "#{@latency_ms}ms" if @latency_ms

data/lib/ruby_llm/contract/token_estimator.rb

@@ -11,9 +11,11 @@ module RubyLLM
     #   - Worse for non-English text, code, structured data, and unusual scripts
     #   - Useless for models with very different tokenizers (e.g. some open-source models)
     #
-    # RubyLLM 1.14 ships no pre-flight tokenizer either; once the API call
-    # returns, `RubyLLM::Tokens` provides accurate counts from provider usage
-    # data. This estimator is for the *pre-flight refusal* path only — its job
+    # RubyLLM 2.x can count tokens exactly before a call (`chat.count_tokens`),
+    # but only on some providers and at the price of an extra request, so this
+    # local estimate stays the default; once the API call returns,
+    # `RubyLLM::Tokens` provides accurate counts from provider usage data.
+    # This estimator is for the *pre-flight refusal* path only - its job
     # is to answer "is this call almost certainly within budget?" with enough
     # accuracy that runaway prompts get caught, while accepting that the
     # boundary cases will be wrong.

data/lib/ruby_llm/contract/unknown_policy.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    # What a limit does when the number it needs is unknown: model pricing for
+    # max_cost/maximum_cost, attachment size for max_input. One vocabulary for
+    # every such option, step-level and eval-level, so a new mode or a changed
+    # default reaches all of them. Standalone: the rake task loads it before the
+    # rest of the gem to validate its settings at definition time.
+    module UnknownPolicy
+      MODES = %i[refuse warn].freeze
+      DEFAULT = :refuse
+
+      def self.validate!(option, mode)
+        return mode if MODES.include?(mode)
+
+        raise ArgumentError, "#{option} must be #{MODES.map(&:inspect).join(" or ")}, got #{mode.inspect}"
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/version.rb

@@ -2,6 +2,6 @@
 
 module RubyLLM
   module Contract
-    VERSION = "1.1.0"
+    VERSION = "1.1.2"
   end
 end

data/lib/ruby_llm/contract.rb

@@ -2,10 +2,19 @@
 
 require_relative "contract/version"
 require_relative "contract/errors"
+require_relative "contract/unknown_policy"
 require_relative "contract/types"
 
 module RubyLLM
   module Contract
+    # Rails dirs holding Step classes. Each one's eval/ subdir holds define_eval
+    # files, which define no constant, so Zeitwerk must ignore exactly those.
+    RAILS_CONTRACT_DIRS = %w[app/contracts app/steps].freeze
+    RAILS_EVAL_DIRS = RAILS_CONTRACT_DIRS.map { |dir| "#{dir}/eval" }.freeze
+
+    # Set while load_evals! runs: redefining an eval is then a reload, not a mistake.
+    RELOADING_KEY = :ruby_llm_contract_reloading
+
     class << self
       def configuration
         @configuration ||= Configuration.new
@@ -51,7 +60,7 @@ module RubyLLM
       def load_evals!(*dirs)
         dirs = dirs.flatten.compact
         if dirs.empty? && defined?(::Rails)
-          dirs = %w[app/steps/eval app/contracts/eval].filter_map do |path|
+          dirs = RAILS_EVAL_DIRS.filter_map do |path|
             full = ::Rails.root.join(path)
             full.to_s if full.exist?
           end
@@ -64,7 +73,7 @@ module RubyLLM
         eager_load_contract_dirs! if defined?(::Rails)
 
         # Clear file-sourced evals ONCE, then load ALL dirs.
-        Thread.current[:ruby_llm_contract_reloading] = true
+        Thread.current[RELOADING_KEY] = true
         eval_hosts.each do |host|
           host.clear_file_sourced_evals! if host.respond_to?(:clear_file_sourced_evals!)
         end
@@ -73,7 +82,11 @@ module RubyLLM
           Dir[File.join(d, "**", "*_eval.rb")].each { |f| load f }
         end
       ensure
-        Thread.current[:ruby_llm_contract_reloading] = false
+        Thread.current[RELOADING_KEY] = false
+      end
+
+      def reloading?
+        Thread.current[RELOADING_KEY] ? true : false
       end
 
       def normalize_candidate_config(entry)
@@ -111,7 +124,7 @@ module RubyLLM
       end
 
       def eager_load_contract_dirs!
-        %w[app/contracts app/steps].each do |path|
+        RAILS_CONTRACT_DIRS.each do |path|
           full = ::Rails.root.join(path)
           next unless full.exist?
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby_llm-contract
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.2
 platform: ruby
 authors:
 - Justyna
@@ -193,6 +193,7 @@ files:
 - lib/ruby_llm/contract/step/trace.rb
 - lib/ruby_llm/contract/token_estimator.rb
 - lib/ruby_llm/contract/types.rb
+- lib/ruby_llm/contract/unknown_policy.rb
 - lib/ruby_llm/contract/version.rb
 - ruby_llm-contract.gemspec
 homepage: https://github.com/justi/ruby_llm-contract