ruby_llm-contract 0.3.7 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dee963c252704634b8b9452e4e0460561e7795385e2dc59f4d5cc089a16d9210
-  data.tar.gz: ce289e0f1dee22a75d7079b28775c6dd0e5d85b01a54e5a97e4f47b40c2f5741
+  metadata.gz: a423ef1b370ae97651d256fdc3776bd895d1eebc81a2b1c4adac305292e2a7a0
+  data.tar.gz: 685ec9b00a369748ca897e38ae498e26d9fc31644aac8c41f096a704bceadd7d
 SHA512:
-  metadata.gz: d10ff4021462051d80cb5205174a24f9c5093ee096fc5add7d5bfacc88fb936a364d474871c05d87dad404ffc9577c998e7a1ae73cc8a8e0a5868e7cef629c83
-  data.tar.gz: 914a370baf65d5e8fc62f78a22e3bc6ee9eba83b78257ac95b87c8d5965ae23e54dbb7a66de7b2b6c7dc3c848a513be22c2e37e76445d9094dd576f3d3867215
+  metadata.gz: 34ab0e678a2de57812a7b8391d406cabe3bb13cf0399669a9bcba18609fc69488d0ef4d2e4a6675436da71fc9b67d3f4c9e264e8fdbef475b07d020d9d8b9d34
+  data.tar.gz: 3aca7548473e4f6e32df442296344013d6081564b56fb5d0081aefc1ea0ab6129896ed852e723c2678548d265797e3568fc4bcf0ca219ca6e220c9ecba35bc9c

data/CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## 0.4.0 (2026-03-24)
+
+Observability & Scale — see what changed, run it fast, debug it easily.
+
+### Features
+
+- **Structured logging** — `Contract.configure { |c| c.logger = Rails.logger }`. Auto-logs model, status, latency, tokens, cost on every `step.run`.
+- **Batch eval concurrency** — `run_eval("regression", concurrency: 4)`. Parallel case execution via Concurrent::Future. 4x faster CI for large eval suites.
+- **Eval history & trending** — `report.save_history!` appends to JSONL. `report.eval_history` returns `EvalHistory` with `score_trend`, `drift?`, run-by-run scores.
+- **Pipeline per-step eval** — `add_case(..., step_expectations: { classify: { priority: "high" } })`. See which step in a pipeline regressed.
+- **Minitest support** — `assert_satisfies_contract`, `assert_eval_passes`, `stub_step` for Minitest users. `require "ruby_llm/contract/minitest"`.
+
+### Game changer continuity
+
+```
+v0.2: "Which model?"          → compare_models (snapshot)
+v0.3: "Did it change?"        → baseline regression (binary)
+v0.4: "Show me the trend"     → eval history (time series)
+      "Which step changed?"   → pipeline per-step eval
+      "Run it fast"           → batch concurrency
+```
+
 ## 0.3.7 (2026-03-24)
 
 - **Trait missing key = error** — `expected_traits: { title: 0..5 }` on output `{}` now fails instead of silently passing.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ruby_llm-contract (0.3.7)
+    ruby_llm-contract (0.4.0)
       dry-types (~> 1.7)
       ruby_llm (~> 1.0)
       ruby_llm-schema (~> 0.3)
@@ -165,7 +165,7 @@ CHECKSUMS
   rubocop-ast (1.49.1) sha256=4412f3ee70f6fe4546cc489548e0f6fcf76cafcfa80fa03af67098ffed755035
   ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33
   ruby_llm (1.14.0) sha256=57c6f7034fc4a44504ea137d70f853b07824f1c1cdbe774ab3ab3522e7098deb
-  ruby_llm-contract (0.3.7)
+  ruby_llm-contract (0.4.0)
   ruby_llm-schema (0.3.0) sha256=a591edc5ca1b7f0304f0e2261de61ba4b3bea17be09f5cf7558153adfda3dec6
   unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42
   unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f

data/README.md

@@ -168,11 +168,11 @@ Works with any ruby_llm provider (OpenAI, Anthropic, Gemini, etc).
 
 ## Roadmap
 
-**v0.3 (current):** Baseline regression detection — `save_baseline!`, `compare_with_baseline`, `without_regressions`. Migration guide.
+**v0.4 (current):** Observability & scale — eval history with trending, batch eval with concurrency, pipeline per-step eval, Minitest support, structured logging.
 
-**v0.2:** Model comparison, cost tracking, eval with `add_case`, CI gating, Rails Railtie.
+**v0.3:** Baseline regression detection, migration guide, production hardening.
 
-**v0.4:** Auto-routing — learn which model works for which input pattern.
+**v0.5:** Prompt A/B testing — `compare_with(OtherStep)` for data-driven prompt engineering with regression safety. Cross-provider comparison docs.
 
 ## License
 

data/lib/ruby_llm/contract/adapters/base.rb

@@ -7,6 +7,12 @@ module RubyLLM
         def call(messages:, **_options)
           raise NotImplementedError, "Subclasses must implement #call"
         end
+
+        # Override in stateful adapters to provide a fully independent copy
+        # for concurrent eval execution. Default: self (stateless adapters).
+        def clone_for_concurrency
+          self
+        end
       end
     end
   end

data/lib/ruby_llm/contract/adapters/test.rb

@@ -29,6 +29,20 @@ module RubyLLM
 
         public
 
+        # Exposes raw responses array for concurrent eval to split per-case
+        def responses_array
+          @responses
+        end
+
+        # Returns a fresh adapter with reset index for concurrent execution
+        def clone_for_concurrency
+          if @responses
+            self.class.new(responses: @responses.dup, usage: @usage.dup)
+          else
+            self.class.new(response: @response, usage: @usage.dup)
+          end
+        end
+
         def call(messages:, **_options) # rubocop:disable Lint/UnusedMethodArgument
           content = if @responses
                       c = @responses[@index] || @responses.last

data/lib/ruby_llm/contract/concerns/context_helpers.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    module Concerns
+      # Shared helpers for context hash manipulation.
+      # Used by EvalHost, Runner, Step::Base.
+      module ContextHelpers
+        private
+
+        def safe_context(context)
+          (context || {}).transform_keys { |k| k.respond_to?(:to_sym) ? k.to_sym : k }
+        end
+
+        def isolate_context(context)
+          context.transform_values do |v|
+            if v.respond_to?(:clone_for_concurrency)
+              v.clone_for_concurrency
+            elsif v.respond_to?(:dup)
+              v.dup
+            else
+              v
+            end
+          rescue TypeError
+            v
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/concerns/eval_host.rb

@@ -4,6 +4,7 @@ module RubyLLM
   module Contract
     module Concerns
       module EvalHost
+        include ContextHelpers
         def define_eval(name, &)
           @eval_definitions ||= {}
           @file_sourced_evals ||= Set.new
@@ -35,20 +36,20 @@ module RubyLLM
           !all_eval_definitions.empty?
         end
 
-        def run_eval(name = nil, context: {})
-          context ||= {}
+        def run_eval(name = nil, context: {}, concurrency: nil)
+          context = safe_context(context)
           if name
-            run_single_eval(name, context)
+            run_single_eval(name, context, concurrency: concurrency)
           else
-            run_all_own_evals(context)
+            run_all_own_evals(context, concurrency: concurrency)
           end
         end
 
         def compare_models(eval_name, models:, context: {})
-          context ||= {}
+          context = safe_context(context)
           models = models.uniq
           reports = models.each_with_object({}) do |model, hash|
-            model_context = deep_dup_context(context).merge(model: model)
+            model_context = isolate_context(context).merge(model: model)
             hash[model] = run_single_eval(eval_name, model_context)
           end
           Eval::ModelComparison.new(eval_name: eval_name, reports: reports)
@@ -66,24 +67,26 @@ module RubyLLM
           inherited.merge(own)
         end
 
-        def run_single_eval(name, context)
+        def run_single_eval(name, context, concurrency: nil)
           defn = all_eval_definitions[name.to_s]
           raise ArgumentError, "No eval '#{name}' defined. Available: #{all_eval_definitions.keys}" unless defn
 
           effective_context = eval_context(defn, context)
-          Eval::Runner.run(step: self, dataset: defn.build_dataset, context: effective_context)
+          Eval::Runner.run(step: self, dataset: defn.build_dataset, context: effective_context,
+                           concurrency: concurrency)
         end
 
-        def run_all_own_evals(context)
+        def run_all_own_evals(context, concurrency: nil)
           all_eval_definitions.transform_values do |defn|
-            isolated_context = deep_dup_context(context)
+            isolated_context = isolate_context(context)
             effective_context = eval_context(defn, isolated_context)
-            Eval::Runner.run(step: self, dataset: defn.build_dataset, context: effective_context)
+            Eval::Runner.run(step: self, dataset: defn.build_dataset, context: effective_context,
+                             concurrency: concurrency)
           end
         end
 
         def eval_context(defn, context)
-          context = (context || {}).transform_keys { |k| k.respond_to?(:to_sym) ? k.to_sym : k }
+          context = safe_context(context)
           return context if context[:adapter]
 
           sample_adapter = defn.build_adapter
@@ -105,13 +108,6 @@ module RubyLLM
           end
         end
 
-        def deep_dup_context(context)
-          context.transform_values do |v|
-            v.respond_to?(:dup) ? v.dup : v
-          rescue TypeError
-            v
-          end
-        end
       end
     end
   end

data/lib/ruby_llm/contract/configuration.rb

@@ -10,11 +10,12 @@ module RubyLLM
     # Then configure contract-specific options:
     #   RubyLLM::Contract.configure { |c| c.default_model = "gpt-4.1-mini" }
     class Configuration
-      attr_accessor :default_adapter, :default_model
+      attr_accessor :default_adapter, :default_model, :logger
 
       def initialize
         @default_adapter = nil
         @default_model = nil
+        @logger = nil
       end
     end
   end

data/lib/ruby_llm/contract/eval/dataset.rb

@@ -22,7 +22,7 @@ module RubyLLM
         # dataset.case "name", input: {...}, expected: {...}
         # dataset.case "name", input: {...}, expected_traits: {...}
         # dataset.case "name", input: {...}, evaluator: proc
-        def add_case(name = nil, input:, expected: nil, expected_traits: nil, evaluator: nil)
+        def add_case(name = nil, input:, expected: nil, expected_traits: nil, evaluator: nil, step_expectations: nil)
           case_name = name || "case_#{@cases.length + 1}"
           if @cases.any? { |c| c.name == case_name }
             raise ArgumentError, "Duplicate case name '#{case_name}'. Case names must be unique within a dataset."
@@ -33,7 +33,8 @@ module RubyLLM
             input: input,
             expected: expected,
             expected_traits: expected_traits,
-            evaluator: evaluator
+            evaluator: evaluator,
+            step_expectations: step_expectations
           )
         end
 
@@ -44,14 +45,15 @@ module RubyLLM
       class Case
         include Concerns::DeepFreeze
 
-        attr_reader :name, :input, :expected, :expected_traits, :evaluator
+        attr_reader :name, :input, :expected, :expected_traits, :evaluator, :step_expectations
 
-        def initialize(name:, input:, expected: nil, expected_traits: nil, evaluator: nil)
+        def initialize(name:, input:, expected: nil, expected_traits: nil, evaluator: nil, step_expectations: nil)
           @name = name
           @input = deep_dup_freeze(input)
           @expected = deep_dup_freeze(expected)
           @expected_traits = deep_dup_freeze(expected_traits)
           @evaluator = evaluator
+          @step_expectations = deep_dup_freeze(step_expectations)
           freeze
         end
       end

data/lib/ruby_llm/contract/eval/eval_definition.rb

@@ -31,7 +31,7 @@ module RubyLLM
           Adapters::Test.new(response: @sample_response)
         end
 
-        def add_case(description, input: nil, expected: nil, expected_traits: nil, evaluator: nil)
+        def add_case(description, input: nil, expected: nil, expected_traits: nil, evaluator: nil, step_expectations: nil)
           case_input = input.nil? ? @default_input : input
           raise ArgumentError, "add_case requires input (set default_input or pass input:)" if case_input.nil?
           validate_unique_case_name!(description)
@@ -41,7 +41,8 @@ module RubyLLM
             input: case_input,
             expected: expected,
             expected_traits: expected_traits,
-            evaluator: evaluator
+            evaluator: evaluator,
+            step_expectations: step_expectations
           }
         end
 
@@ -72,7 +73,8 @@ module RubyLLM
             eval_cases.each do |eval_case|
               add_case(eval_case[:name], input: eval_case[:input], expected: eval_case[:expected],
                                          expected_traits: eval_case[:expected_traits],
-                                         evaluator: eval_case[:evaluator])
+                                         evaluator: eval_case[:evaluator],
+                                         step_expectations: eval_case[:step_expectations])
             end
           end
         end

data/lib/ruby_llm/contract/eval/eval_history.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+module RubyLLM
+  module Contract
+    module Eval
+      class EvalHistory
+        attr_reader :runs
+
+        def initialize(runs:)
+          @runs = runs.freeze
+          freeze
+        end
+
+        def self.load(path)
+          return new(runs: []) unless File.exist?(path)
+
+          runs = File.readlines(path).filter_map do |line|
+            JSON.parse(line.strip, symbolize_names: true)
+          rescue JSON::ParserError
+            nil
+          end
+          new(runs: runs)
+        end
+
+        def self.append(path, run_data)
+          FileUtils.mkdir_p(File.dirname(path))
+          File.open(path, "a") { |f| f.puts(run_data.to_json) }
+        end
+
+        def score_trend
+          return :unknown if runs.length < 2
+
+          scores = runs.map { |r| r[:score] }
+          recent = scores.last(3)
+          if recent.all? { |s| s >= scores.first }
+            :stable_or_improving
+          elsif recent.last < scores.max * 0.9
+            :declining
+          else
+            :stable_or_improving
+          end
+        end
+
+        def drift?(threshold: 0.1)
+          return false if runs.length < 2
+
+          baseline_score = runs.first[:score]
+          current_score = runs.last[:score]
+          (baseline_score - current_score) > threshold
+        end
+
+        def scores
+          runs.map { |r| r[:score] }
+        end
+
+        def dates
+          runs.map { |r| r[:date] }
+        end
+
+        def latest
+          runs.last
+        end
+
+        def to_s
+          return "No history" if runs.empty?
+
+          lines = ["#{runs.length} runs"]
+          runs.last(5).each do |r|
+            lines << "  #{r[:date]} score=#{r[:score].round(2)} cost=$#{format("%.6f", r[:total_cost] || r[:cost] || 0)}"
+          end
+          lines.join("\n")
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/eval/report.rb

@@ -82,6 +82,24 @@ module RubyLLM
           lines.join("\n")
         end
 
+        def save_history!(path: nil, model: nil)
+          file = path || default_history_path(model: model)
+          run_data = {
+            date: Time.now.strftime("%Y-%m-%d"),
+            score: score,
+            total_cost: total_cost,
+            pass_rate: pass_rate,
+            cases_count: evaluated_results.length
+          }
+          EvalHistory.append(file, run_data)
+          file
+        end
+
+        def eval_history(path: nil, model: nil)
+          file = path || default_history_path(model: model)
+          EvalHistory.load(file)
+        end
+
         def save_baseline!(path: nil, model: nil)
           file = path || default_baseline_path(model: model)
           FileUtils.mkdir_p(File.dirname(file))
@@ -133,6 +151,15 @@ module RubyLLM
           results.reject { |r| r.step_status == :skipped }
         end
 
+        def default_history_path(model: nil)
+          parts = [".eval_history"]
+          parts << sanitize_name(@step_name) if @step_name
+          name = sanitize_name(dataset_name)
+          name = "#{name}_#{sanitize_name(model)}" if model
+          parts << "#{name}.jsonl"
+          File.join(*parts)
+        end
+
         def default_baseline_path(model: nil)
           parts = [".eval_baselines"]
           parts << sanitize_name(@step_name) if @step_name

data/lib/ruby_llm/contract/eval/runner.rb

@@ -6,31 +6,100 @@ module RubyLLM
       class Runner
         include TraitEvaluator
         include ContractDetailBuilder
+        include Concerns::ContextHelpers
 
-        def self.run(step:, dataset:, context: {})
-          new(step: step, dataset: dataset, context: context).run
+        def self.run(step:, dataset:, context: {}, concurrency: nil)
+          new(step: step, dataset: dataset, context: context, concurrency: concurrency).run
         end
 
-        def initialize(step:, dataset:, context: {})
+        def initialize(step:, dataset:, context: {}, concurrency: nil)
           @step = step
           @dataset = dataset
           @context = context
+          @concurrency = concurrency
         end
 
         def run
-          results = @dataset.cases.map { |test_case| evaluate_case(test_case) }
+          results = if @concurrency && @concurrency > 1
+                      run_concurrent
+                    else
+                      @dataset.cases.map { |test_case| evaluate_case(test_case) }
+                    end
           step_name = @step.respond_to?(:name) ? @step.name : @step.to_s
           Report.new(dataset_name: @dataset.name, results: results, step_name: step_name)
         end
 
         private
 
+        def run_concurrent
+          require "concurrent"
+          pool = Concurrent::FixedThreadPool.new(@concurrency)
+
+          # Pre-build per-case contexts: if adapter has responses:, each case
+          # gets a single-response adapter with its own response (by index).
+          per_case_contexts = build_per_case_contexts
+
+          futures = @dataset.cases.each_with_index.map do |test_case, i|
+            ctx = per_case_contexts[i]
+            Concurrent::Future.execute(executor: pool) do
+              evaluate_case_with_context(test_case, ctx)
+            end
+          end
+          futures.map(&:value!)
+        ensure
+          pool&.shutdown
+          pool&.wait_for_termination(5)
+        end
+
+        def build_per_case_contexts
+          adapter = @context[:adapter]
+          responses = adapter.respond_to?(:responses_array) ? adapter.responses_array : nil
+
+          @dataset.cases.each_with_index.map do |_, i|
+            if responses
+              # Give each case its own single-response adapter
+              response = responses[i] || responses.last
+              per_case_adapter = Adapters::Test.new(response: response)
+              @context.merge(adapter: per_case_adapter)
+            else
+              isolate_context(@context)
+            end
+          end
+        end
+
+        def evaluate_case_with_context(test_case, context)
+          run_result = @step.run(test_case.input, context: context)
+          step_result = normalize_result(run_result)
+          eval_result = dispatch_evaluation(step_result, test_case)
+
+          result = build_case_result(test_case, step_result, eval_result)
+
+          if test_case.respond_to?(:step_expectations) && test_case.step_expectations &&
+             run_result.respond_to?(:outputs_by_step)
+            evaluate_step_expectations(result, run_result.outputs_by_step, test_case.step_expectations)
+          else
+            result
+          end
+        rescue RubyLLM::Contract::Error => e
+          raise unless e.message.include?("No adapter configured")
+
+          skipped_result(test_case, e.message)
+        end
+
         def evaluate_case(test_case)
           run_result = @step.run(test_case.input, context: @context)
           step_result = normalize_result(run_result)
           eval_result = dispatch_evaluation(step_result, test_case)
 
-          build_case_result(test_case, step_result, eval_result)
+          result = build_case_result(test_case, step_result, eval_result)
+
+          # Pipeline per-step evaluation
+          if test_case.respond_to?(:step_expectations) && test_case.step_expectations &&
+             run_result.respond_to?(:outputs_by_step)
+            evaluate_step_expectations(result, run_result.outputs_by_step, test_case.step_expectations)
+          else
+            result
+          end
         rescue RubyLLM::Contract::Error => e
           raise unless e.message.include?("No adapter configured")
 
@@ -145,6 +214,38 @@ module RubyLLM
           )
         end
 
+        def evaluate_step_expectations(result, outputs_by_step, expectations)
+          step_results = {}
+          all_passed = true
+
+          expectations.each do |step_alias, expected|
+            output = outputs_by_step[step_alias]
+            if output.nil?
+              step_results[step_alias] = { passed: false, details: "step not executed" }
+              all_passed = false
+            else
+              eval_res = dispatch_expected_evaluator(output: output, expected: expected, input: nil)
+              step_results[step_alias] = { passed: eval_res.passed, score: eval_res.score, details: eval_res.details }
+              all_passed = false unless eval_res.passed
+            end
+          end
+
+          # Rebuild CaseResult with step_results metadata
+          failed_steps = step_results.select { |_, v| !v[:passed] }
+          failure_details = failed_steps.map { |k, v| "#{k}: #{v[:details]}" }.join("; ")
+
+          CaseResult.new(
+            name: result.name, input: result.input, output: result.output,
+            expected: result.expected,
+            step_status: all_passed ? result.step_status : :step_expectation_failed,
+            score: all_passed ? result.score : 0.0,
+            passed: result.passed? && all_passed,
+            label: all_passed ? result.label : "FAIL",
+            details: all_passed ? result.details : "step expectations failed: #{failure_details}",
+            duration_ms: result.duration_ms, cost: result.cost
+          )
+        end
+
         def skipped_result(test_case, reason)
           CaseResult.new(
             name: test_case.name,

data/lib/ruby_llm/contract/eval.rb

@@ -15,3 +15,4 @@ require_relative "eval/report"
 require_relative "eval/eval_definition"
 require_relative "eval/model_comparison"
 require_relative "eval/baseline_diff"
+require_relative "eval/eval_history"

data/lib/ruby_llm/contract/minitest.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require "ruby_llm/contract"
+
+module RubyLLM
+  module Contract
+    module MinitestHelpers
+      def assert_satisfies_contract(result, msg = nil)
+        assert result.ok?, msg || "Expected step result to satisfy contract, " \
+          "but got status: #{result.status}. Errors: #{result.validation_errors.join(", ")}"
+      end
+
+      def refute_satisfies_contract(result, msg = nil)
+        refute result.ok?, msg || "Expected step result NOT to satisfy contract, but it passed"
+      end
+
+      def assert_eval_passes(step, eval_name, minimum_score: nil, maximum_cost: nil, context: {}, msg: nil)
+        report = step.run_eval(eval_name, context: context)
+
+        if minimum_score
+          assert report.score >= minimum_score,
+                 msg || "Expected #{eval_name} eval score >= #{minimum_score}, got #{report.score.round(2)} (#{report.pass_rate})"
+        else
+          assert report.passed?,
+                 msg || "Expected #{eval_name} eval to pass, got #{report.score.round(2)} (#{report.pass_rate})"
+        end
+
+        if maximum_cost
+          assert report.total_cost <= maximum_cost,
+                 msg || "Expected #{eval_name} eval cost <= $#{format("%.4f", maximum_cost)}, got $#{format("%.4f", report.total_cost)}"
+        end
+
+        report
+      end
+
+      def stub_step(step_class, response: nil, responses: nil)
+        adapter = if responses
+                    Adapters::Test.new(responses: responses)
+                  else
+                    Adapters::Test.new(response: response)
+                  end
+        RubyLLM::Contract.configure { |c| c.default_adapter = adapter }
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/railtie.rb

@@ -3,15 +3,21 @@
 module RubyLLM
   module Contract
     class Railtie < ::Rails::Railtie
-      # Eval files (e.g. classify_threads_eval.rb) don't define Zeitwerk-compatible
-      # constants — they call define_eval on an existing Step class. We use `load`
-      # after initialization, and hook into the reloader for development.
+      # Ignore eval/ subdirs BEFORE Zeitwerk setup — eval files don't define
+      # constants, they call define_eval on existing Step classes.
+      initializer "ruby_llm_contract.ignore_eval_dirs", before: :set_autoload_paths do |app|
+        %w[app/contracts/eval app/steps/eval].each do |path|
+          full = app.root.join(path)
+          next unless full.exist?
+
+          Rails.autoloaders.each { |loader| loader.ignore(full.to_s) }
+        end
+      end
 
       config.after_initialize do
         RubyLLM::Contract.load_evals!
       end
 
-      # Re-load eval files on code reload in development (Spring, zeitwerk:check, etc.)
       config.to_prepare do
         RubyLLM::Contract.load_evals!
       end

data/lib/ruby_llm/contract/step/base.rb

@@ -60,8 +60,10 @@ module RubyLLM
 
           KNOWN_CONTEXT_KEYS = %i[adapter model temperature max_tokens provider assume_model_exists].freeze
 
+          include Concerns::ContextHelpers
+
           def run(input, context: {})
-            context = (context || {}).transform_keys { |k| k.respond_to?(:to_sym) ? k.to_sym : k }
+            context = safe_context(context)
             warn_unknown_context_keys(context)
             adapter = resolve_adapter(context)
             default_model = context[:model] || model || RubyLLM::Contract.configuration.default_model
@@ -77,6 +79,7 @@ module RubyLLM
                                 context_temperature: ctx_temp, extra_options: extra)
                      end
 
+            log_result(result)
             invoke_around_call(input, result)
           end
 
@@ -121,6 +124,19 @@ module RubyLLM
                        validation_errors: [e.message])
           end
 
+          def log_result(result)
+            logger = RubyLLM::Contract.configuration.logger
+            return unless logger
+
+            trace = result.trace
+            msg = "[ruby_llm-contract] #{name || self} " \
+                  "model=#{trace.model} status=#{result.status} " \
+                  "latency=#{trace.latency_ms}ms " \
+                  "tokens=#{trace.usage&.dig(:input_tokens) || 0}+#{trace.usage&.dig(:output_tokens) || 0} " \
+                  "cost=$#{format("%.6f", trace.cost || 0)}"
+            logger.info(msg)
+          end
+
           def invoke_around_call(input, result)
             return result unless around_call
 

data/lib/ruby_llm/contract/version.rb

@@ -2,6 +2,6 @@
 
 module RubyLLM
   module Contract
-    VERSION = "0.3.7"
+    VERSION = "0.4.0"
   end
 end

data/lib/ruby_llm/contract.rb

@@ -88,10 +88,7 @@ module RubyLLM
           full = ::Rails.root.join(path)
           next unless full.exist?
 
-          # Ignore eval/ subdirs — they don't define Zeitwerk-compatible
-          # constants and are loaded separately by load_evals!
-          eval_dir = full.join("eval")
-          ::Rails.autoloaders.main.ignore(eval_dir.to_s) if eval_dir.exist?
+          # eval/ subdirs already ignored by Railtie initializer (before Zeitwerk setup)
           ::Rails.autoloaders.main.eager_load_dir(full.to_s)
         rescue StandardError
           nil
@@ -108,6 +105,7 @@ module RubyLLM
   end
 end
 
+require_relative "contract/concerns/context_helpers"
 require_relative "contract/concerns/deep_freeze"
 require_relative "contract/concerns/deep_symbolize"
 require_relative "contract/concerns/eval_host"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ruby_llm-contract
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.4.0
 platform: ruby
 authors:
 - Justyna
@@ -83,6 +83,7 @@ files:
 - lib/ruby_llm/contract/adapters/response.rb
 - lib/ruby_llm/contract/adapters/ruby_llm.rb
 - lib/ruby_llm/contract/adapters/test.rb
+- lib/ruby_llm/contract/concerns/context_helpers.rb
 - lib/ruby_llm/contract/concerns/deep_freeze.rb
 - lib/ruby_llm/contract/concerns/deep_symbolize.rb
 - lib/ruby_llm/contract/concerns/eval_host.rb
@@ -104,6 +105,7 @@ files:
 - lib/ruby_llm/contract/eval/contract_detail_builder.rb
 - lib/ruby_llm/contract/eval/dataset.rb
 - lib/ruby_llm/contract/eval/eval_definition.rb
+- lib/ruby_llm/contract/eval/eval_history.rb
 - lib/ruby_llm/contract/eval/evaluation_result.rb
 - lib/ruby_llm/contract/eval/evaluator/exact.rb
 - lib/ruby_llm/contract/eval/evaluator/json_includes.rb
@@ -114,6 +116,7 @@ files:
 - lib/ruby_llm/contract/eval/report.rb
 - lib/ruby_llm/contract/eval/runner.rb
 - lib/ruby_llm/contract/eval/trait_evaluator.rb
+- lib/ruby_llm/contract/minitest.rb
 - lib/ruby_llm/contract/pipeline.rb
 - lib/ruby_llm/contract/pipeline/base.rb
 - lib/ruby_llm/contract/pipeline/result.rb