ruby_llm-contract 0.4.2 → 0.5.0

data/lib/ruby_llm/contract/step/limit_checker.rb

@@ -9,7 +9,7 @@ module RubyLLM
         private
 
         def check_limits(messages)
-          return nil unless @max_input || @max_cost
+          return nil unless max_input || max_cost
 
           estimated = TokenEstimator.estimate(messages)
           errors = collect_limit_errors(estimated)
@@ -21,27 +21,44 @@ module RubyLLM
 
         def collect_limit_errors(estimated)
           errors = []
-          if @max_input && estimated > @max_input
-            errors << "Input token limit exceeded: estimated #{estimated} tokens, max #{@max_input}"
+          if max_input && estimated > max_input
+            errors << "Input token limit exceeded: estimated #{estimated} tokens, max #{max_input}"
           end
-          append_cost_error(estimated, errors) if @max_cost
+          append_cost_error(estimated, errors) if max_cost
           errors
         end
 
+        # Default output estimate when max_output is not set.
+        # Uses input token count as a conservative proxy — most LLM responses
+        # are shorter than the input, so this overestimates slightly.
+        # Without this, output cost is zero and max_cost can be bypassed
+        # for models expensive on completion side.
+        DEFAULT_OUTPUT_RATIO = 1
+
         def append_cost_error(estimated, errors)
-          estimated_output = effective_max_output || 0
+          estimated_output = effective_max_output || (estimated * DEFAULT_OUTPUT_RATIO)
           estimated_cost = CostCalculator.calculate(
-            model_name: @model,
+            model_name: model_name,
             usage: { input_tokens: estimated, output_tokens: estimated_output }
           )
 
           if estimated_cost.nil?
-            warn "[ruby_llm-contract] max_cost is configured but model '#{@model}' " \
-                 "has no pricing data — cost limit not enforced"
-          elsif estimated_cost > @max_cost
+            handle_unknown_pricing(errors)
+          elsif estimated_cost > max_cost
             errors << "Cost limit exceeded: estimated $#{format("%.6f", estimated_cost)} " \
                       "(#{estimated} input + #{estimated_output} output tokens), " \
-                      "max $#{format("%.6f", @max_cost)}"
+                      "max $#{format("%.6f", max_cost)}"
+          end
+        end
+
+        def handle_unknown_pricing(errors)
+          if on_unknown_pricing == :warn
+            warn "[ruby_llm-contract] max_cost is configured but model '#{model_name}' " \
+                 "has no pricing data — cost limit not enforced"
+          else
+            errors << "max_cost is set but model '#{model_name}' has no pricing data. " \
+                      "Register pricing via CostCalculator.register_model or set " \
+                      "on_unknown_pricing: :warn to proceed without cost checks."
           end
         end
 
@@ -52,7 +69,7 @@ module RubyLLM
             parsed_output: nil,
             validation_errors: errors,
             trace: Trace.new(
-              messages: messages, model: @model,
+              messages: messages, model: model_name,
               usage: { input_tokens: 0, output_tokens: 0, estimated_input_tokens: estimated,
                        estimate_method: :heuristic }
             )

data/lib/ruby_llm/contract/step/prompt_compiler.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Step
+      class PromptCompiler
+        def initialize(prompt_block:)
+          @prompt_block = prompt_block
+        end
+
+        def call(input)
+          dynamic_prompt = @prompt_block.arity >= 1
+          builder_input = dynamic_prompt ? input : nil
+          ast = Prompt::Builder.build(input: builder_input, &@prompt_block)
+
+          Prompt::Renderer.render(ast, variables: template_variables_for(input, dynamic_prompt))
+        rescue StandardError => error
+          raise RubyLLM::Contract::Error, "Prompt build failed: #{error.class}: #{error.message}"
+        end
+
+        private
+
+        def template_variables_for(input, dynamic_prompt)
+          return {} if dynamic_prompt
+
+          { input: input }.tap do |variables|
+            variables.merge!(input.transform_keys(&:to_sym)) if input.is_a?(Hash)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/step/result.rb

@@ -4,13 +4,14 @@ module RubyLLM
   module Contract
     module Step
       class Result
-        attr_reader :status, :raw_output, :parsed_output, :validation_errors, :trace
+        attr_reader :status, :raw_output, :parsed_output, :validation_errors, :trace, :observations
 
-        def initialize(status:, raw_output:, parsed_output:, validation_errors: [], trace: nil)
+        def initialize(status:, raw_output:, parsed_output:, validation_errors: [], trace: nil, observations: [])
           @status = status
           @raw_output = raw_output
           @parsed_output = parsed_output
           @validation_errors = validation_errors.freeze
+          @observations = observations.freeze
           @trace = normalize_trace(trace)
           freeze
         end

data/lib/ruby_llm/contract/step/result_builder.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Step
+      class ResultBuilder
+        def initialize(contract_definition:, output_type:, output_schema:, model:, observers:)
+          @contract_definition = contract_definition
+          @output_type = output_type
+          @output_schema = output_schema
+          @model = model
+          @observers = observers
+        end
+
+        def error_result(error_result:, messages:)
+          Result.new(
+            status: error_result.status,
+            raw_output: error_result.raw_output,
+            parsed_output: error_result.parsed_output,
+            validation_errors: error_result.validation_errors,
+            trace: Trace.new(messages: messages, model: @model)
+          )
+        end
+
+        def success_result(response:, messages:, latency_ms:, input:)
+          raw_output = response.content
+          validation_result = validate_output(raw_output, input)
+          trace = Trace.new(messages: messages, model: @model, latency_ms: latency_ms, usage: response.usage)
+
+          Result.new(
+            status: validation_result[:status],
+            raw_output: raw_output,
+            parsed_output: validation_result[:parsed_output],
+            validation_errors: validation_result[:errors],
+            trace: trace,
+            observations: observations_for(validation_result, input)
+          )
+        end
+
+        private
+
+        def observations_for(validation_result, input)
+          return [] unless validation_result[:status] == :ok && @observers.any?
+
+          Validator.run_observations(@observers, validation_result[:parsed_output], input: input)
+        end
+
+        def validate_output(raw_output, input)
+          Validator.validate(
+            raw_output: raw_output,
+            definition: @contract_definition,
+            output_type: @output_type,
+            input: input,
+            schema: @output_schema
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/step/retry_executor.rb

@@ -32,6 +32,7 @@ module RubyLLM
           Result.new(
             status: last.status, raw_output: last.raw_output,
             parsed_output: last.parsed_output, validation_errors: last.validation_errors,
+            observations: last.observations,
             trace: last.trace.merge(
               attempts: attempt_log, usage: aggregated_usage,
               cost: total_cost, latency_ms: total_latency

data/lib/ruby_llm/contract/step/runner.rb

@@ -8,26 +8,31 @@ module RubyLLM
 
         def initialize(input_type:, output_type:, prompt_block:, contract_definition:,
                        adapter:, model:, output_schema: nil, max_output: nil,
-                       max_input: nil, max_cost: nil, temperature: nil, extra_options: {})
-          @input_type = input_type
-          @output_type = output_type
-          @prompt_block = prompt_block
-          @contract_definition = contract_definition
-          @adapter = adapter
-          @model = model
-          @output_schema = output_schema
-          @max_output = max_output
-          @max_input = max_input
-          @max_cost = max_cost
-          @temperature = temperature
-          @extra_options = extra_options
+                       max_input: nil, max_cost: nil, on_unknown_pricing: :refuse,
+                       temperature: nil, extra_options: {}, observers: [])
+          @config = RunnerConfig.new(
+            input_type: input_type,
+            output_type: output_type,
+            prompt_block: prompt_block,
+            contract_definition: contract_definition,
+            adapter: adapter,
+            model: model,
+            output_schema: output_schema,
+            max_output: max_output,
+            max_input: max_input,
+            max_cost: max_cost,
+            on_unknown_pricing: on_unknown_pricing,
+            temperature: temperature,
+            extra_options: extra_options,
+            observers: observers
+          )
         end
 
         def call(input)
-          validated_input = validate_input(input)
+          validated_input = input_validator.call(input)
           return validated_input if validated_input.is_a?(Result)
 
-          messages = build_and_render_prompt(input)
+          messages = prompt_compiler.call(input)
         rescue RubyLLM::Contract::Error => e
           Result.new(status: :input_error, raw_output: nil, parsed_output: nil,
                      validation_errors: [e.message])
@@ -41,94 +46,52 @@ module RubyLLM
           limit_result = check_limits(messages)
           return limit_result if limit_result
 
-          response, latency_ms = execute_adapter(messages)
-          return build_error_result(response, messages) if response.is_a?(Result)
+          response, latency_ms = adapter_caller.call(messages)
+          return result_builder.error_result(error_result: response, messages: messages) if response.is_a?(Result)
 
-          build_result(response, messages, latency_ms, input)
+          result_builder.success_result(response: response, messages: messages, latency_ms: latency_ms, input: input)
         end
 
-        def validate_input(input)
-          type = @input_type
-          if type.is_a?(Class) && !type.respond_to?(:[])
-            raise TypeError, "#{input.inspect} is not a #{type}" unless input.is_a?(type)
-          else
-            type[input]
-          end
-          nil
-        rescue Dry::Types::CoercionError, TypeError, ArgumentError => e
-          Result.new(status: :input_error, raw_output: nil, parsed_output: nil, validation_errors: [e.message])
+        def input_validator
+          InputValidator.new(input_type: @config.input_type)
         end
 
-        def build_and_render_prompt(input)
-          dynamic = @prompt_block.arity >= 1
-          ast = Prompt::Builder.build(input: dynamic ? input : nil, &@prompt_block)
-
-          Prompt::Renderer.render(ast, variables: dynamic ? {} : template_variables_for(input))
-        rescue StandardError => e
-          raise RubyLLM::Contract::Error, "Prompt build failed: #{e.class}: #{e.message}"
+        def prompt_compiler
+          PromptCompiler.new(prompt_block: @config.prompt_block)
         end
 
-        def template_variables_for(input)
-          base = { input: input }
-          input.is_a?(Hash) ? base.merge(input.transform_keys(&:to_sym)) : base
+        def adapter_caller
+          AdapterCaller.new(adapter: @config.adapter, adapter_options: @config.adapter_options)
         end
 
-        def execute_adapter(messages)
-          start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-          response = @adapter.call(messages: messages, **build_adapter_options)
-          latency_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
-          [response, latency_ms]
-        rescue StandardError => e
-          [Result.new(status: :adapter_error, raw_output: nil, parsed_output: nil, validation_errors: [e.message]), 0]
+        def result_builder
+          ResultBuilder.new(
+            contract_definition: @config.contract_definition,
+            output_type: @config.output_type,
+            output_schema: @config.output_schema,
+            model: @config.model,
+            observers: @config.observers
+          )
         end
 
-        def build_adapter_options
-          effective_max_tokens = @extra_options[:max_tokens] || @max_output
-
-          { model: @model }.tap do |opts|
-            opts[:schema] = @output_schema if @output_schema
-            opts[:max_tokens] = effective_max_tokens if effective_max_tokens
-            opts[:temperature] = @temperature if @temperature
-            @extra_options.each { |k, v| opts[k] = v unless opts.key?(k) }
-          end
+        def max_input
+          @config.max_input
         end
 
-        def effective_max_output
-          @extra_options[:max_tokens] || @max_output
+        def max_cost
+          @config.max_cost
         end
 
-        def build_error_result(error_result, messages)
-          Result.new(
-            status: error_result.status,
-            raw_output: error_result.raw_output,
-            parsed_output: error_result.parsed_output,
-            validation_errors: error_result.validation_errors,
-            trace: Trace.new(messages: messages, model: @model)
-          )
+        def model_name
+          @config.model
         end
 
-        def build_result(response, messages, latency_ms, input)
-          raw_output = response.content
-          validation_result = validate_output(raw_output, input)
-          trace = Trace.new(messages: messages, model: @model, latency_ms: latency_ms, usage: response.usage)
-
-          Result.new(
-            status: validation_result[:status],
-            raw_output: raw_output,
-            parsed_output: validation_result[:parsed_output],
-            validation_errors: validation_result[:errors],
-            trace: trace
-          )
+        def on_unknown_pricing
+          @config.on_unknown_pricing
         end
 
-        def validate_output(raw_output, input)
-          Validator.validate(
-            raw_output: raw_output,
-            definition: @contract_definition,
-            output_type: @output_type,
-            input: input,
-            schema: @output_schema
-          )
+        def effective_max_output
+          @config.effective_max_output
         end
       end
     end

data/lib/ruby_llm/contract/step/runner_config.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Step
+      RunnerConfig = Data.define(
+        :input_type,
+        :output_type,
+        :prompt_block,
+        :contract_definition,
+        :adapter,
+        :model,
+        :output_schema,
+        :max_output,
+        :max_input,
+        :max_cost,
+        :on_unknown_pricing,
+        :temperature,
+        :extra_options,
+        :observers
+      ) do
+        def effective_max_output
+          extra_options[:max_tokens] || max_output
+        end
+
+        def adapter_options
+          { model: model }.tap do |options|
+            options[:schema] = output_schema if output_schema
+            options[:max_tokens] = effective_max_output if effective_max_output
+            options[:temperature] = temperature if temperature
+            extra_options.each { |key, value| options[key] = value unless options.key?(key) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/step.rb

@@ -3,6 +3,11 @@
 require_relative "step/trace"
 require_relative "step/result"
 require_relative "step/limit_checker"
+require_relative "step/runner_config"
+require_relative "step/input_validator"
+require_relative "step/prompt_compiler"
+require_relative "step/adapter_caller"
+require_relative "step/result_builder"
 require_relative "step/runner"
 require_relative "step/retry_policy"
 require_relative "step/retry_executor"

data/lib/ruby_llm/contract/version.rb

@@ -2,6 +2,6 @@
 
 module RubyLLM
   module Contract
-    VERSION = "0.4.2"
+    VERSION = "0.5.0"
   end
 end

data/lib/ruby_llm/contract.rb

@@ -18,6 +18,7 @@ module RubyLLM
 
       def reset_configuration!
         @configuration = Configuration.new
+        step_adapter_overrides.clear
       end
 
       # --- Eval host registry ---
@@ -40,6 +41,15 @@ module RubyLLM
         @eval_hosts = []
       end
 
+      # Thread-local per-step adapter overrides used by test helpers (RSpec + Minitest).
+      def step_adapter_overrides
+        Thread.current[:ruby_llm_contract_step_overrides] ||= {}
+      end
+
+      def step_adapter_overrides=(map)
+        Thread.current[:ruby_llm_contract_step_overrides] = map
+      end
+
       def load_evals!(*dirs)
         dirs = dirs.flatten.compact
         if dirs.empty? && defined?(::Rails)
@@ -102,6 +112,21 @@ module RubyLLM
         nil
       end
     end
+
+    # One-time prepend on Step::Base that checks the override map before
+    # falling through to the normal adapter resolution.
+    # Used by both RSpec and Minitest test helpers.
+    module StepAdapterOverride
+      def run(input, context: {})
+        context = context || {}
+        overrides = RubyLLM::Contract.step_adapter_overrides
+        unless overrides.empty? || context.key?(:adapter) || context.key?("adapter")
+          override = overrides[self]
+          context = context.merge(adapter: override) if override
+        end
+        super(input, context: context)
+      end
+    end
   end
 end
 
@@ -126,3 +151,6 @@ require_relative "contract/pipeline"
 require_relative "contract/eval"
 require_relative "contract/dsl"
 require_relative "contract/railtie" if defined?(Rails::Railtie)
+
+# Prepend after Step::Base is loaded
+RubyLLM::Contract::Step::Base.singleton_class.prepend(RubyLLM::Contract::StepAdapterOverride)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby_llm-contract
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
 platform: ruby
 authors:
 - Justyna
@@ -60,6 +60,8 @@ extra_rdoc_files: []
 files:
 - ".rspec"
 - ".rubocop.yml"
+- ".rubycritic.yml"
+- ".simplecov"
 - CHANGELOG.md
 - Gemfile
 - Gemfile.lock
@@ -95,13 +97,23 @@ files:
 - lib/ruby_llm/contract/contract/invariant.rb
 - lib/ruby_llm/contract/contract/parser.rb
 - lib/ruby_llm/contract/contract/schema_validator.rb
+- lib/ruby_llm/contract/contract/schema_validator/bound_rule.rb
+- lib/ruby_llm/contract/contract/schema_validator/enum_rule.rb
+- lib/ruby_llm/contract/contract/schema_validator/node.rb
+- lib/ruby_llm/contract/contract/schema_validator/object_rules.rb
+- lib/ruby_llm/contract/contract/schema_validator/scalar_rules.rb
+- lib/ruby_llm/contract/contract/schema_validator/schema_extractor.rb
+- lib/ruby_llm/contract/contract/schema_validator/type_rule.rb
 - lib/ruby_llm/contract/contract/validator.rb
 - lib/ruby_llm/contract/cost_calculator.rb
 - lib/ruby_llm/contract/dsl.rb
 - lib/ruby_llm/contract/errors.rb
 - lib/ruby_llm/contract/eval.rb
 - lib/ruby_llm/contract/eval/baseline_diff.rb
+- lib/ruby_llm/contract/eval/case_executor.rb
 - lib/ruby_llm/contract/eval/case_result.rb
+- lib/ruby_llm/contract/eval/case_result_builder.rb
+- lib/ruby_llm/contract/eval/case_scorer.rb
 - lib/ruby_llm/contract/eval/contract_detail_builder.rb
 - lib/ruby_llm/contract/eval/dataset.rb
 - lib/ruby_llm/contract/eval/eval_definition.rb
@@ -111,10 +123,20 @@ files:
 - lib/ruby_llm/contract/eval/evaluator/json_includes.rb
 - lib/ruby_llm/contract/eval/evaluator/proc_evaluator.rb
 - lib/ruby_llm/contract/eval/evaluator/regex.rb
+- lib/ruby_llm/contract/eval/expectation_evaluator.rb
 - lib/ruby_llm/contract/eval/model_comparison.rb
 - lib/ruby_llm/contract/eval/pipeline_result_adapter.rb
+- lib/ruby_llm/contract/eval/prompt_diff.rb
+- lib/ruby_llm/contract/eval/prompt_diff_comparator.rb
+- lib/ruby_llm/contract/eval/prompt_diff_presenter.rb
+- lib/ruby_llm/contract/eval/prompt_diff_serializer.rb
 - lib/ruby_llm/contract/eval/report.rb
+- lib/ruby_llm/contract/eval/report_presenter.rb
+- lib/ruby_llm/contract/eval/report_stats.rb
+- lib/ruby_llm/contract/eval/report_storage.rb
 - lib/ruby_llm/contract/eval/runner.rb
+- lib/ruby_llm/contract/eval/step_expectation_applier.rb
+- lib/ruby_llm/contract/eval/step_result_normalizer.rb
 - lib/ruby_llm/contract/eval/trait_evaluator.rb
 - lib/ruby_llm/contract/minitest.rb
 - lib/ruby_llm/contract/pipeline.rb
@@ -139,13 +161,18 @@ files:
 - lib/ruby_llm/contract/rspec/pass_eval.rb
 - lib/ruby_llm/contract/rspec/satisfy_contract.rb
 - lib/ruby_llm/contract/step.rb
+- lib/ruby_llm/contract/step/adapter_caller.rb
 - lib/ruby_llm/contract/step/base.rb
 - lib/ruby_llm/contract/step/dsl.rb
+- lib/ruby_llm/contract/step/input_validator.rb
 - lib/ruby_llm/contract/step/limit_checker.rb
+- lib/ruby_llm/contract/step/prompt_compiler.rb
 - lib/ruby_llm/contract/step/result.rb
+- lib/ruby_llm/contract/step/result_builder.rb
 - lib/ruby_llm/contract/step/retry_executor.rb
 - lib/ruby_llm/contract/step/retry_policy.rb
 - lib/ruby_llm/contract/step/runner.rb
+- lib/ruby_llm/contract/step/runner_config.rb
 - lib/ruby_llm/contract/step/trace.rb
 - lib/ruby_llm/contract/token_estimator.rb
 - lib/ruby_llm/contract/types.rb