ruby_llm-contract 0.4.5 → 0.5.0

data/lib/ruby_llm/contract/step/base.rb

@@ -4,6 +4,8 @@ module RubyLLM
   module Contract
     module Step
       class Base
+        DEFAULT_OUTPUT_TOKENS = 256
+
         def self.inherited(subclass)
           super
           Contract.register_eval_host(subclass) if respond_to?(:eval_defined?) && eval_defined?
@@ -15,30 +17,23 @@ module RubyLLM
           include Dsl
 
           def eval_case(input:, expected: nil, expected_traits: nil, evaluator: nil, context: {})
-            dataset = Eval::Dataset.define("single_case") do
-              add_case("inline", input: input, expected: expected,
-                                 expected_traits: expected_traits, evaluator: evaluator)
-            end
-            report = Eval::Runner.run(step: self, dataset: dataset, context: context)
-            report.results.first
+            Eval::Runner.run(step: self, dataset: inline_dataset(input, expected, expected_traits, evaluator),
+                             context: context).results.first
           end
 
           def estimate_cost(input:, model: nil)
-            model_name = model || (self.model if respond_to?(:model)) || RubyLLM::Contract.configuration.default_model
-            messages = build_messages(input)
-            input_tokens = TokenEstimator.estimate(messages)
-            output_tokens = max_output || 256 # conservative default
-
+            model_name = estimated_model_name(model)
             model_info = CostCalculator.send(:find_model, model_name)
             return nil unless model_info
 
-            estimated = CostCalculator.send(:compute_cost, model_info,
-                                            { input_tokens: input_tokens, output_tokens: output_tokens })
+            input_tokens = TokenEstimator.estimate(build_messages(input))
+            output_tokens = max_output || DEFAULT_OUTPUT_TOKENS
+
             {
               model: model_name,
               input_tokens: input_tokens,
               output_tokens_estimate: output_tokens,
-              estimated_cost: estimated
+              estimated_cost: estimated_cost_for(model_info, input_tokens, output_tokens)
             }
           end
 
@@ -46,16 +41,11 @@ module RubyLLM
             defn = send(:all_eval_definitions)[eval_name.to_s]
             raise ArgumentError, "No eval '#{eval_name}' defined" unless defn
 
-            step_model = (self.model if respond_to?(:model))
-            model_list = models || [step_model || RubyLLM::Contract.configuration.default_model].compact
+            model_list = models || [estimated_model_name].compact
             cases = defn.build_dataset.cases
 
             model_list.each_with_object({}) do |model_name, result|
-              per_case = cases.sum do |c|
-                est = estimate_cost(input: c.input, model: model_name)
-                est ? est[:estimated_cost] : 0.0
-              end
-              result[model_name] = per_case.round(6)
+              result[model_name] = estimate_eval_cost_for_model(cases, model_name)
             end
           end
 
@@ -66,20 +56,8 @@ module RubyLLM
           def run(input, context: {})
             context = safe_context(context)
             warn_unknown_context_keys(context)
-            adapter = resolve_adapter(context)
-            default_model = context[:model] || model || RubyLLM::Contract.configuration.default_model
-            policy = retry_policy
-
-            ctx_temp = context[:temperature]
-            extra = context.slice(:provider, :assume_model_exists, :max_tokens)
-            result = if policy
-                       run_with_retry(input, adapter: adapter, default_model: default_model,
-                                      policy: policy, context_temperature: ctx_temp, extra_options: extra)
-                     else
-                       run_once(input, adapter: adapter, model: default_model,
-                                context_temperature: ctx_temp, extra_options: extra)
-                     end
 
+            result = dispatch_run(input, context)
             log_result(result)
             invoke_around_call(input, result)
           end
@@ -88,13 +66,43 @@ module RubyLLM
             dynamic = prompt.arity >= 1
             builder_input = dynamic ? input : Prompt::Builder::NOT_PROVIDED
             ast = Prompt::Builder.build(input: builder_input, &prompt)
-            variables = dynamic ? {} : { input: input }
-            variables.merge!(input.transform_keys(&:to_sym)) if !dynamic && input.is_a?(Hash)
-            Prompt::Renderer.render(ast, variables: variables)
+            Prompt::Renderer.render(ast, variables: prompt_variables(input, dynamic))
           end
 
           private
 
+          def inline_dataset(input, expected, expected_traits, evaluator)
+            Eval::Dataset.define("single_case") do
+              add_case("inline", input: input, expected: expected,
+                                 expected_traits: expected_traits, evaluator: evaluator)
+            end
+          end
+
+          def estimated_model_name(model = nil)
+            model || (self.model if respond_to?(:model)) || RubyLLM::Contract.configuration.default_model
+          end
+
+          def estimated_cost_for(model_info, input_tokens, output_tokens)
+            CostCalculator.send(
+              :compute_cost,
+              model_info,
+              { input_tokens: input_tokens, output_tokens: output_tokens }
+            )
+          end
+
+          def estimate_eval_cost_for_model(cases, model_name)
+            cases.sum do |test_case|
+              estimate = estimate_cost(input: test_case.input, model: model_name)
+              estimate ? estimate[:estimated_cost] : 0.0
+            end.round(6)
+          end
+
+          def prompt_variables(input, dynamic)
+            variables = dynamic ? {} : { input: input }
+            variables.merge!(input.transform_keys(&:to_sym)) if !dynamic && input.is_a?(Hash)
+            variables
+          end
+
           def warn_unknown_context_keys(context)
             unknown = context.keys - KNOWN_CONTEXT_KEYS
             return if unknown.empty?
@@ -103,6 +111,39 @@ module RubyLLM
                  "Known keys: #{KNOWN_CONTEXT_KEYS.inspect}"
           end
 
+          def dispatch_run(input, context)
+            adapter = resolve_adapter(context)
+            runtime = runtime_settings(context)
+
+            if runtime[:policy]
+              run_with_retry(
+                input,
+                adapter: adapter,
+                default_model: runtime[:model],
+                policy: runtime[:policy],
+                context_temperature: runtime[:temperature],
+                extra_options: runtime[:extra_options]
+              )
+            else
+              run_once(
+                input,
+                adapter: adapter,
+                model: runtime[:model],
+                context_temperature: runtime[:temperature],
+                extra_options: runtime[:extra_options]
+              )
+            end
+          end
+
+          def runtime_settings(context)
+            {
+              model: context[:model] || model || RubyLLM::Contract.configuration.default_model,
+              temperature: context[:temperature],
+              extra_options: context.slice(:provider, :assume_model_exists, :max_tokens),
+              policy: retry_policy
+            }
+          end
+
           def resolve_adapter(context)
             adapter = context[:adapter] || RubyLLM::Contract.configuration.default_adapter
             return adapter if adapter
@@ -119,7 +160,8 @@ module RubyLLM
               adapter: adapter, model: model, output_schema: output_schema,
               max_output: max_output, max_input: max_input, max_cost: max_cost,
               on_unknown_pricing: on_unknown_pricing,
-              temperature: effective_temp, extra_options: extra_options
+              temperature: effective_temp, extra_options: extra_options,
+              observers: class_observers
             ).call(input)
           rescue ArgumentError => e
             Result.new(status: :input_error, raw_output: nil, parsed_output: nil,
@@ -137,6 +179,19 @@ module RubyLLM
                   "tokens=#{trace.usage&.dig(:input_tokens) || 0}+#{trace.usage&.dig(:output_tokens) || 0} " \
                   "cost=$#{format("%.6f", trace.cost || 0)}"
             logger.info(msg)
+
+            log_failed_observations(result, logger)
+          end
+
+          def log_failed_observations(result, logger)
+            failed = result.observations.select { |o| !o[:passed] }
+            return if failed.empty?
+
+            failed.each do |obs|
+              msg = "[ruby_llm-contract] #{name || self} observation failed: #{obs[:description]}"
+              msg += " (#{obs[:error]})" if obs[:error]
+              logger.warn(msg)
+            end
           end
 
           def invoke_around_call(input, result)

data/lib/ruby_llm/contract/step/dsl.rb

@@ -79,6 +79,16 @@ module RubyLLM
           inherited + own
         end
 
+        def observe(description, &block)
+          (@class_observers ||= []) << Invariant.new(description, block)
+        end
+
+        def class_observers
+          own = defined?(@class_observers) ? @class_observers : []
+          inherited = superclass.respond_to?(:class_observers) ? superclass.class_observers : []
+          inherited + own
+        end
+
         def max_output(tokens = nil)
           if tokens
             unless tokens.is_a?(Numeric) && tokens.positive?

data/lib/ruby_llm/contract/step/input_validator.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Step
+      class InputValidator
+        def initialize(input_type:)
+          @input_type = input_type
+        end
+
+        def call(input)
+          validate(input)
+          nil
+        rescue Dry::Types::CoercionError, TypeError, ArgumentError => error
+          Result.new(status: :input_error, raw_output: nil, parsed_output: nil, validation_errors: [error.message])
+        end
+
+        private
+
+        def validate(input)
+          if ruby_class_input?
+            raise TypeError, "#{input.inspect} is not a #{@input_type}" unless input.is_a?(@input_type)
+          else
+            @input_type[input]
+          end
+        end
+
+        def ruby_class_input?
+          @input_type.is_a?(Class) && !@input_type.respond_to?(:[])
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/step/limit_checker.rb

@@ -9,7 +9,7 @@ module RubyLLM
         private
 
         def check_limits(messages)
-          return nil unless @max_input || @max_cost
+          return nil unless max_input || max_cost
 
           estimated = TokenEstimator.estimate(messages)
           errors = collect_limit_errors(estimated)
@@ -21,10 +21,10 @@ module RubyLLM
 
         def collect_limit_errors(estimated)
           errors = []
-          if @max_input && estimated > @max_input
-            errors << "Input token limit exceeded: estimated #{estimated} tokens, max #{@max_input}"
+          if max_input && estimated > max_input
+            errors << "Input token limit exceeded: estimated #{estimated} tokens, max #{max_input}"
           end
-          append_cost_error(estimated, errors) if @max_cost
+          append_cost_error(estimated, errors) if max_cost
           errors
         end
 
@@ -38,25 +38,25 @@ module RubyLLM
         def append_cost_error(estimated, errors)
           estimated_output = effective_max_output || (estimated * DEFAULT_OUTPUT_RATIO)
           estimated_cost = CostCalculator.calculate(
-            model_name: @model,
+            model_name: model_name,
             usage: { input_tokens: estimated, output_tokens: estimated_output }
           )
 
           if estimated_cost.nil?
             handle_unknown_pricing(errors)
-          elsif estimated_cost > @max_cost
+          elsif estimated_cost > max_cost
             errors << "Cost limit exceeded: estimated $#{format("%.6f", estimated_cost)} " \
                       "(#{estimated} input + #{estimated_output} output tokens), " \
-                      "max $#{format("%.6f", @max_cost)}"
+                      "max $#{format("%.6f", max_cost)}"
           end
         end
 
         def handle_unknown_pricing(errors)
-          if @on_unknown_pricing == :warn
-            warn "[ruby_llm-contract] max_cost is configured but model '#{@model}' " \
+          if on_unknown_pricing == :warn
+            warn "[ruby_llm-contract] max_cost is configured but model '#{model_name}' " \
                  "has no pricing data — cost limit not enforced"
           else
-            errors << "max_cost is set but model '#{@model}' has no pricing data. " \
+            errors << "max_cost is set but model '#{model_name}' has no pricing data. " \
                       "Register pricing via CostCalculator.register_model or set " \
                       "on_unknown_pricing: :warn to proceed without cost checks."
           end
@@ -69,7 +69,7 @@ module RubyLLM
             parsed_output: nil,
             validation_errors: errors,
             trace: Trace.new(
-              messages: messages, model: @model,
+              messages: messages, model: model_name,
               usage: { input_tokens: 0, output_tokens: 0, estimated_input_tokens: estimated,
                        estimate_method: :heuristic }
             )

data/lib/ruby_llm/contract/step/prompt_compiler.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Step
+      class PromptCompiler
+        def initialize(prompt_block:)
+          @prompt_block = prompt_block
+        end
+
+        def call(input)
+          dynamic_prompt = @prompt_block.arity >= 1
+          builder_input = dynamic_prompt ? input : nil
+          ast = Prompt::Builder.build(input: builder_input, &@prompt_block)
+
+          Prompt::Renderer.render(ast, variables: template_variables_for(input, dynamic_prompt))
+        rescue StandardError => error
+          raise RubyLLM::Contract::Error, "Prompt build failed: #{error.class}: #{error.message}"
+        end
+
+        private
+
+        def template_variables_for(input, dynamic_prompt)
+          return {} if dynamic_prompt
+
+          { input: input }.tap do |variables|
+            variables.merge!(input.transform_keys(&:to_sym)) if input.is_a?(Hash)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/step/result.rb

@@ -4,13 +4,14 @@ module RubyLLM
   module Contract
     module Step
       class Result
-        attr_reader :status, :raw_output, :parsed_output, :validation_errors, :trace
+        attr_reader :status, :raw_output, :parsed_output, :validation_errors, :trace, :observations
 
-        def initialize(status:, raw_output:, parsed_output:, validation_errors: [], trace: nil)
+        def initialize(status:, raw_output:, parsed_output:, validation_errors: [], trace: nil, observations: [])
           @status = status
           @raw_output = raw_output
           @parsed_output = parsed_output
           @validation_errors = validation_errors.freeze
+          @observations = observations.freeze
           @trace = normalize_trace(trace)
           freeze
         end

data/lib/ruby_llm/contract/step/result_builder.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Step
+      class ResultBuilder
+        def initialize(contract_definition:, output_type:, output_schema:, model:, observers:)
+          @contract_definition = contract_definition
+          @output_type = output_type
+          @output_schema = output_schema
+          @model = model
+          @observers = observers
+        end
+
+        def error_result(error_result:, messages:)
+          Result.new(
+            status: error_result.status,
+            raw_output: error_result.raw_output,
+            parsed_output: error_result.parsed_output,
+            validation_errors: error_result.validation_errors,
+            trace: Trace.new(messages: messages, model: @model)
+          )
+        end
+
+        def success_result(response:, messages:, latency_ms:, input:)
+          raw_output = response.content
+          validation_result = validate_output(raw_output, input)
+          trace = Trace.new(messages: messages, model: @model, latency_ms: latency_ms, usage: response.usage)
+
+          Result.new(
+            status: validation_result[:status],
+            raw_output: raw_output,
+            parsed_output: validation_result[:parsed_output],
+            validation_errors: validation_result[:errors],
+            trace: trace,
+            observations: observations_for(validation_result, input)
+          )
+        end
+
+        private
+
+        def observations_for(validation_result, input)
+          return [] unless validation_result[:status] == :ok && @observers.any?
+
+          Validator.run_observations(@observers, validation_result[:parsed_output], input: input)
+        end
+
+        def validate_output(raw_output, input)
+          Validator.validate(
+            raw_output: raw_output,
+            definition: @contract_definition,
+            output_type: @output_type,
+            input: input,
+            schema: @output_schema
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/step/retry_executor.rb

@@ -32,6 +32,7 @@ module RubyLLM
           Result.new(
             status: last.status, raw_output: last.raw_output,
             parsed_output: last.parsed_output, validation_errors: last.validation_errors,
+            observations: last.observations,
             trace: last.trace.merge(
               attempts: attempt_log, usage: aggregated_usage,
               cost: total_cost, latency_ms: total_latency

data/lib/ruby_llm/contract/step/runner.rb

@@ -9,27 +9,30 @@ module RubyLLM
         def initialize(input_type:, output_type:, prompt_block:, contract_definition:,
                        adapter:, model:, output_schema: nil, max_output: nil,
                        max_input: nil, max_cost: nil, on_unknown_pricing: :refuse,
-                       temperature: nil, extra_options: {})
-          @input_type = input_type
-          @output_type = output_type
-          @prompt_block = prompt_block
-          @contract_definition = contract_definition
-          @adapter = adapter
-          @model = model
-          @output_schema = output_schema
-          @max_output = max_output
-          @max_input = max_input
-          @max_cost = max_cost
-          @on_unknown_pricing = on_unknown_pricing
-          @temperature = temperature
-          @extra_options = extra_options
+                       temperature: nil, extra_options: {}, observers: [])
+          @config = RunnerConfig.new(
+            input_type: input_type,
+            output_type: output_type,
+            prompt_block: prompt_block,
+            contract_definition: contract_definition,
+            adapter: adapter,
+            model: model,
+            output_schema: output_schema,
+            max_output: max_output,
+            max_input: max_input,
+            max_cost: max_cost,
+            on_unknown_pricing: on_unknown_pricing,
+            temperature: temperature,
+            extra_options: extra_options,
+            observers: observers
+          )
         end
 
         def call(input)
-          validated_input = validate_input(input)
+          validated_input = input_validator.call(input)
           return validated_input if validated_input.is_a?(Result)
 
-          messages = build_and_render_prompt(input)
+          messages = prompt_compiler.call(input)
         rescue RubyLLM::Contract::Error => e
           Result.new(status: :input_error, raw_output: nil, parsed_output: nil,
                      validation_errors: [e.message])
@@ -43,94 +46,52 @@ module RubyLLM
           limit_result = check_limits(messages)
           return limit_result if limit_result
 
-          response, latency_ms = execute_adapter(messages)
-          return build_error_result(response, messages) if response.is_a?(Result)
+          response, latency_ms = adapter_caller.call(messages)
+          return result_builder.error_result(error_result: response, messages: messages) if response.is_a?(Result)
 
-          build_result(response, messages, latency_ms, input)
+          result_builder.success_result(response: response, messages: messages, latency_ms: latency_ms, input: input)
         end
 
-        def validate_input(input)
-          type = @input_type
-          if type.is_a?(Class) && !type.respond_to?(:[])
-            raise TypeError, "#{input.inspect} is not a #{type}" unless input.is_a?(type)
-          else
-            type[input]
-          end
-          nil
-        rescue Dry::Types::CoercionError, TypeError, ArgumentError => e
-          Result.new(status: :input_error, raw_output: nil, parsed_output: nil, validation_errors: [e.message])
+        def input_validator
+          InputValidator.new(input_type: @config.input_type)
         end
 
-        def build_and_render_prompt(input)
-          dynamic = @prompt_block.arity >= 1
-          ast = Prompt::Builder.build(input: dynamic ? input : nil, &@prompt_block)
-
-          Prompt::Renderer.render(ast, variables: dynamic ? {} : template_variables_for(input))
-        rescue StandardError => e
-          raise RubyLLM::Contract::Error, "Prompt build failed: #{e.class}: #{e.message}"
+        def prompt_compiler
+          PromptCompiler.new(prompt_block: @config.prompt_block)
         end
 
-        def template_variables_for(input)
-          base = { input: input }
-          input.is_a?(Hash) ? base.merge(input.transform_keys(&:to_sym)) : base
+        def adapter_caller
+          AdapterCaller.new(adapter: @config.adapter, adapter_options: @config.adapter_options)
         end
 
-        def execute_adapter(messages)
-          start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-          response = @adapter.call(messages: messages, **build_adapter_options)
-          latency_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round
-          [response, latency_ms]
-        rescue StandardError => e
-          [Result.new(status: :adapter_error, raw_output: nil, parsed_output: nil, validation_errors: [e.message]), 0]
+        def result_builder
+          ResultBuilder.new(
+            contract_definition: @config.contract_definition,
+            output_type: @config.output_type,
+            output_schema: @config.output_schema,
+            model: @config.model,
+            observers: @config.observers
+          )
         end
 
-        def build_adapter_options
-          effective_max_tokens = @extra_options[:max_tokens] || @max_output
-
-          { model: @model }.tap do |opts|
-            opts[:schema] = @output_schema if @output_schema
-            opts[:max_tokens] = effective_max_tokens if effective_max_tokens
-            opts[:temperature] = @temperature if @temperature
-            @extra_options.each { |k, v| opts[k] = v unless opts.key?(k) }
-          end
+        def max_input
+          @config.max_input
         end
 
-        def effective_max_output
-          @extra_options[:max_tokens] || @max_output
+        def max_cost
+          @config.max_cost
         end
 
-        def build_error_result(error_result, messages)
-          Result.new(
-            status: error_result.status,
-            raw_output: error_result.raw_output,
-            parsed_output: error_result.parsed_output,
-            validation_errors: error_result.validation_errors,
-            trace: Trace.new(messages: messages, model: @model)
-          )
+        def model_name
+          @config.model
         end
 
-        def build_result(response, messages, latency_ms, input)
-          raw_output = response.content
-          validation_result = validate_output(raw_output, input)
-          trace = Trace.new(messages: messages, model: @model, latency_ms: latency_ms, usage: response.usage)
-
-          Result.new(
-            status: validation_result[:status],
-            raw_output: raw_output,
-            parsed_output: validation_result[:parsed_output],
-            validation_errors: validation_result[:errors],
-            trace: trace
-          )
+        def on_unknown_pricing
+          @config.on_unknown_pricing
         end
 
-        def validate_output(raw_output, input)
-          Validator.validate(
-            raw_output: raw_output,
-            definition: @contract_definition,
-            output_type: @output_type,
-            input: input,
-            schema: @output_schema
-          )
+        def effective_max_output
+          @config.effective_max_output
         end
       end
     end

data/lib/ruby_llm/contract/step/runner_config.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Step
+      RunnerConfig = Data.define(
+        :input_type,
+        :output_type,
+        :prompt_block,
+        :contract_definition,
+        :adapter,
+        :model,
+        :output_schema,
+        :max_output,
+        :max_input,
+        :max_cost,
+        :on_unknown_pricing,
+        :temperature,
+        :extra_options,
+        :observers
+      ) do
+        def effective_max_output
+          extra_options[:max_tokens] || max_output
+        end
+
+        def adapter_options
+          { model: model }.tap do |options|
+            options[:schema] = output_schema if output_schema
+            options[:max_tokens] = effective_max_output if effective_max_output
+            options[:temperature] = temperature if temperature
+            extra_options.each { |key, value| options[key] = value unless options.key?(key) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/step.rb

@@ -3,6 +3,11 @@
 require_relative "step/trace"
 require_relative "step/result"
 require_relative "step/limit_checker"
+require_relative "step/runner_config"
+require_relative "step/input_validator"
+require_relative "step/prompt_compiler"
+require_relative "step/adapter_caller"
+require_relative "step/result_builder"
 require_relative "step/runner"
 require_relative "step/retry_policy"
 require_relative "step/retry_executor"

data/lib/ruby_llm/contract/version.rb

@@ -2,6 +2,6 @@
 
 module RubyLLM
   module Contract
-    VERSION = "0.4.5"
+    VERSION = "0.5.0"
   end
 end