ruby_llm-contract 0.8.0 → 0.10.1

data/lib/ruby_llm/contract/minitest.rb

@@ -5,6 +5,8 @@ require "ruby_llm/contract"
 module RubyLLM
   module Contract
     module MinitestHelpers
+      include Concerns::StubHelpers
+
       # Snapshot adapter before each test so teardown can restore it.
       def setup
         super if defined?(super)
@@ -47,114 +49,10 @@ module RubyLLM
         report
       end
 
-      # Stub a specific step to return a canned response without API calls.
-      # Routes per-step — other steps are not affected.
-      #
-      #   stub_step(ClassifyTicket, response: { priority: "high" })
-      #
-      # Supports an optional block form — the override is removed after the
-      # block returns (even if it raises):
-      #
-      #   stub_step(ClassifyTicket, response: data) do
-      #     result = ClassifyTicket.run("test")
-      #   end
-      #   # ClassifyTicket.run no longer stubbed
-      #
-      def stub_step(step_class, response: nil, responses: nil, &block)
-        adapter = if responses
-                    Adapters::Test.new(responses: responses)
-                  else
-                    Adapters::Test.new(response: response)
-                  end
-
-        overrides = RubyLLM::Contract.step_adapter_overrides
-        previous = overrides[step_class]
-        overrides[step_class] = adapter
-
-        if block
-          begin
-            yield
-          ensure
-            if previous
-              overrides[step_class] = previous
-            else
-              overrides.delete(step_class)
-            end
-          end
-        end
-      end
-
-      # Stub multiple steps at once with different responses.
-      # Takes a hash of step_class => options. Requires a block.
-      #
-      #   stub_steps(
-      #     ClassifyTicket => { response: { priority: "high" } },
-      #     RouteToTeam => { response: { team: "billing" } }
-      #   ) do
-      #     result = TicketPipeline.run("test")
-      #   end
-      #
-      def stub_steps(stubs, &block)
-        raise ArgumentError, "stub_steps requires a block" unless block
-
-        overrides = RubyLLM::Contract.step_adapter_overrides
-        previous = {}
-
-        stubs.each do |step_class, opts|
-          opts = opts.transform_keys(&:to_sym)
-          adapter = if opts[:responses]
-                      Adapters::Test.new(responses: opts[:responses])
-                    else
-                      Adapters::Test.new(response: opts[:response])
-                    end
-          previous[step_class] = overrides[step_class]
-          overrides[step_class] = adapter
-        end
-
-        begin
-          yield
-        ensure
-          stubs.each_key do |step_class|
-            if previous[step_class]
-              overrides[step_class] = previous[step_class]
-            else
-              overrides.delete(step_class)
-            end
-          end
-        end
-      end
-
-      # Set a global test adapter for ALL steps.
-      #
-      #   stub_all_steps(response: { default: true })
-      #
-      # Supports an optional block form — the previous adapter is restored
-      # after the block returns (even if it raises):
-      #
-      #   stub_all_steps(response: { default: true }) do
-      #     # all steps use test adapter
-      #   end
-      #   # original adapter restored
-      #
-      def stub_all_steps(response: nil, responses: nil, &block)
-        adapter = if responses
-                    Adapters::Test.new(responses: responses)
-                  else
-                    Adapters::Test.new(response: response)
-                  end
-
-        if block
-          previous = RubyLLM::Contract.configuration.default_adapter
-          begin
-            RubyLLM::Contract.configuration.default_adapter = adapter
-            yield
-          ensure
-            RubyLLM::Contract.configuration.default_adapter = previous
-          end
-        else
-          RubyLLM::Contract.configure { |c| c.default_adapter = adapter }
-        end
-      end
+      # `stub_step`, `stub_steps`, `stub_all_steps` — provided by
+      # `Concerns::StubHelpers` (included above). Shared implementation
+      # used by both Minitest and RSpec hosts; documentation and method
+      # signatures live in `concerns/stub_helpers.rb`.
     end
   end
 end

data/lib/ruby_llm/contract/pipeline/result.rb

@@ -108,7 +108,7 @@ module RubyLLM
           trace = step_result.trace
           status = step_status(step_result)
           trace_str = trace.respond_to?(:to_s) ? trace.to_s : ""
-          "  #{step_record[:alias].to_s.ljust(14)} #{status.ljust(10)} #{trace_str}"
+          "  #{step_record[:alias].to_s.ljust(COL1)} #{status.ljust(COL2)} #{trace_str}"
         end
 
         def step_status(step_result)

data/lib/ruby_llm/contract/rake_task/suite_gate.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module RubyLLM
+  module Contract
+    class RakeTask < ::Rake::TaskLib
+      # Encapsulates the pass/fail gate that runs after `RakeTask#define_task`
+      # has collected eval reports. Extracted from the prior `define_task`
+      # god-method so each gating dimension (cost, score, regression) is
+      # testable in isolation.
+      #
+      # Returns a `Verdict` value object with:
+      #   - `passed?`         — overall gate verdict
+      #   - `abort_reason`    — String for `abort` when `passed? == false`, nil otherwise
+      #   - `passed_reports`  — [[host, report], ...] of reports that individually passed
+      #                         (used to decide which baselines to save)
+      #   - `suite_cost`      — total cost across all reports
+      #
+      # Gate ordering (preserved from pre-refactor behaviour):
+      #   1. cost gate runs FIRST — if `maximum_cost` set and exceeded, the
+      #      suite aborts before any score check; passed_reports is empty.
+      #   2. score gate runs per-report; a report passes if
+      #      `report_meets_score?` AND `!check_regression`.
+      #   3. overall passed = ALL reports passed AND cost gate not tripped.
+      class SuiteGate
+        Verdict = Data.define(:passed, :abort_reason, :passed_reports, :suite_cost) do
+          def passed?
+            passed
+          end
+        end
+
+        def self.evaluate(host_reports:, minimum_score:, maximum_cost:, fail_on_regression:)
+          new(host_reports: host_reports,
+              minimum_score: minimum_score,
+              maximum_cost: maximum_cost,
+              fail_on_regression: fail_on_regression).verdict
+        end
+
+        attr_reader :verdict
+
+        def initialize(host_reports:, minimum_score:, maximum_cost:, fail_on_regression:)
+          @host_reports = host_reports
+          @minimum_score = minimum_score
+          @maximum_cost = maximum_cost
+          @fail_on_regression = fail_on_regression
+          @verdict = build_verdict
+        end
+
+        private
+
+        def build_verdict
+          suite_cost = compute_suite_cost
+
+          if cost_exceeded?(suite_cost)
+            return Verdict.new(
+              passed: false,
+              abort_reason: cost_abort_message(suite_cost),
+              passed_reports: [],
+              suite_cost: suite_cost
+            )
+          end
+
+          passed_reports, all_passed = score_each_report
+          Verdict.new(
+            passed: all_passed,
+            abort_reason: all_passed ? nil : "Eval suite FAILED",
+            passed_reports: passed_reports,
+            suite_cost: suite_cost
+          )
+        end
+
+        def compute_suite_cost
+          @host_reports.sum { |_host, report| report.total_cost }
+        end
+
+        def cost_exceeded?(suite_cost)
+          @maximum_cost && suite_cost > @maximum_cost
+        end
+
+        def cost_abort_message(suite_cost)
+          "total cost $#{format("%.4f", suite_cost)} exceeds budget $#{format("%.4f", @maximum_cost)}"
+        end
+
+        def score_each_report
+          passed_reports = []
+          all_passed = true
+          @host_reports.each do |host, report|
+            report_ok = report_meets_score?(report) && !check_regression(report)
+            all_passed = false unless report_ok
+            passed_reports << [host, report] if report_ok
+          end
+          [passed_reports, all_passed]
+        end
+
+        def report_meets_score?(report)
+          if @minimum_score
+            report.score >= @minimum_score
+          else
+            report.passed?
+          end
+        end
+
+        def check_regression(report)
+          return false unless @fail_on_regression && report.baseline_exists?
+
+          diff = report.compare_with_baseline
+          if diff.regressed?
+            puts "\n  REGRESSIONS DETECTED:"
+            puts "  #{diff}"
+            true
+          else
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/contract/rake_task.rb

@@ -2,6 +2,7 @@
 
 require "rake"
 require "rake/tasklib"
+require_relative "rake_task/suite_gate"
 
 module RubyLLM
   module Contract
@@ -33,71 +34,49 @@ module RubyLLM
           RubyLLM::Contract.load_evals!(*@eval_dirs)
 
           context = @context.respond_to?(:call) ? @context.call : @context
-          results = RubyLLM::Contract.run_all_evals(context: context)
-
-          if results.empty?
-            if @fail_on_empty
-              abort "No evals defined. Define evals with define_eval or set fail_on_empty = false."
-            else
-              puts "No evals defined."
-              next
-            end
-          end
-
-          gate_passed = true
-          suite_cost = 0.0
-
-          passed_reports = []
-          all_reports = []
-
-          results.each do |host, reports|
-            puts "\n#{host.name || host.to_s}"
-            reports.each_value do |report|
-              report.print_summary
-              suite_cost += report.total_cost
-              all_reports << [host, report]
-              report_ok = report_meets_score?(report) && !check_regression(report)
-              gate_passed = false unless report_ok
-              passed_reports << report if report_ok
-            end
-          end
+          host_reports = collect_host_reports(context)
+          next unless host_reports # empty path already handled
 
           # Save history BEFORE gating — failures are valuable trend data (ADR-0016 F3)
-          save_all_history!(all_reports, context) if @track_history
+          save_all_history!(host_reports, context) if @track_history
 
-          if @maximum_cost && suite_cost > @maximum_cost
-            abort "\nEval suite FAILED: total cost $#{format("%.4f", suite_cost)} " \
-                  "exceeds budget $#{format("%.4f", @maximum_cost)}"
-          end
+          verdict = SuiteGate.evaluate(
+            host_reports: host_reports,
+            minimum_score: @minimum_score,
+            maximum_cost: @maximum_cost,
+            fail_on_regression: @fail_on_regression
+          )
 
-          abort "\nEval suite FAILED" unless gate_passed
+          abort "\nEval suite FAILED: #{verdict.abort_reason}" unless verdict.passed?
 
           # Save baselines only after ALL gates pass
-          passed_reports.each { |r| save_baseline!(r) } if @save_baseline
+          verdict.passed_reports.each { |_host, r| save_baseline!(r) } if @save_baseline
 
           puts "\nAll evals passed."
         end
       end
 
-      def report_meets_score?(report)
-        if @minimum_score
-          report.score >= @minimum_score
-        else
-          report.passed?
-        end
-      end
+      def collect_host_reports(context)
+        results = RubyLLM::Contract.run_all_evals(context: context)
 
-      def check_regression(report)
-        return false unless @fail_on_regression && report.baseline_exists?
+        if results.empty?
+          if @fail_on_empty
+            abort "No evals defined. Define evals with define_eval or set fail_on_empty = false."
+          else
+            puts "No evals defined."
+            return nil
+          end
+        end
 
-        diff = report.compare_with_baseline
-        if diff.regressed?
-          puts "\n  REGRESSIONS DETECTED:"
-          puts "  #{diff}"
-          true
-        else
-          false
+        host_reports = []
+        results.each do |host, reports|
+          puts "\n#{host.name || host.to_s}"
+          reports.each_value do |report|
+            report.print_summary
+            host_reports << [host, report]
+          end
         end
+        host_reports
       end
 
       def save_baseline!(report)

data/lib/ruby_llm/contract/rspec/helpers.rb

@@ -3,130 +3,16 @@
 module RubyLLM
   module Contract
     module RSpec
+      # `stub_step`, `stub_steps`, `stub_all_steps` — provided by
+      # `Concerns::StubHelpers`. Shared implementation used by both RSpec
+      # and Minitest hosts; documentation and method signatures live in
+      # `concerns/stub_helpers.rb`.
+      #
+      # Cleanup between examples is handled by the `around(:each)` hook
+      # in `lib/ruby_llm/contract/rspec.rb`, which snapshots and restores
+      # `step_adapter_overrides` plus `configuration.default_adapter`.
       module Helpers
-        # Stub a step to return a canned response without API calls.
-        #
-        #   stub_step(ClassifyTicket, response: { priority: "high" })
-        #   result = ClassifyTicket.run("test")
-        #   result.parsed_output  # => {priority: "high"}
-        #
-        # Only affects the specified step — other steps are not affected.
-        #
-        # With a block, the stub is scoped — cleaned up after the block:
-        #
-        #   stub_step(ClassifyTicket, response: data) do
-        #     # only stubbed inside this block
-        #   end
-        #   # ClassifyTicket no longer stubbed
-        #
-        # Without a block, the stub lives until the RSpec example ends.
-        #
-        def stub_step(step_class, response: nil, responses: nil, &block)
-          adapter = build_test_adapter(response: response, responses: responses)
-
-          if block
-            # Block form: use thread-local overrides with save/restore for real scoping
-            overrides = RubyLLM::Contract.step_adapter_overrides
-            previous = overrides[step_class]
-            overrides[step_class] = adapter
-            begin
-              yield
-            ensure
-              if previous
-                overrides[step_class] = previous
-              else
-                overrides.delete(step_class)
-              end
-            end
-          else
-            # Non-block: use RSpec allow (auto-cleaned after example)
-            allow(step_class).to receive(:run).and_wrap_original do |original, input, **kwargs|
-              context = kwargs[:context] || {}
-              unless context.key?(:adapter) || context.key?("adapter")
-                context = context.merge(adapter: adapter)
-              end
-              original.call(input, context: context)
-            end
-          end
-        end
-
-        # Stub multiple steps at once with different responses.
-        # Takes a hash of step_class => options. Requires a block.
-        #
-        #   stub_steps(
-        #     ClassifyTicket => { response: { priority: "high" } },
-        #     RouteToTeam => { response: { team: "billing" } }
-        #   ) do
-        #     result = TicketPipeline.run("test")
-        #   end
-        #
-        def stub_steps(stubs, &block)
-          raise ArgumentError, "stub_steps requires a block" unless block
-
-          overrides = RubyLLM::Contract.step_adapter_overrides
-          previous = {}
-
-          stubs.each do |step_class, opts|
-            opts = opts.transform_keys(&:to_sym)
-            adapter = build_test_adapter(**opts)
-            previous[step_class] = overrides[step_class]
-            overrides[step_class] = adapter
-          end
-
-          begin
-            yield
-          ensure
-            stubs.each_key do |step_class|
-              if previous[step_class]
-                overrides[step_class] = previous[step_class]
-              else
-                overrides.delete(step_class)
-              end
-            end
-          end
-        end
-
-        # Set a global test adapter for ALL steps.
-        #
-        #   stub_all_steps(response: { default: true })
-        #
-        # Supports an optional block form — the previous adapter is restored
-        # after the block returns (even if it raises):
-        #
-        #   stub_all_steps(response: { default: true }) do
-        #     # all steps use test adapter
-        #   end
-        #   # original adapter restored
-        #
-        def stub_all_steps(response: nil, responses: nil, &block)
-          adapter = build_test_adapter(response: response, responses: responses)
-
-          if block
-            previous = RubyLLM::Contract.configuration.default_adapter
-            begin
-              RubyLLM::Contract.configuration.default_adapter = adapter
-              yield
-            ensure
-              RubyLLM::Contract.configuration.default_adapter = previous
-            end
-          else
-            RubyLLM::Contract.configure { |c| c.default_adapter = adapter }
-          end
-        end
-
-        private
-
-        def build_test_adapter(response: nil, responses: nil)
-          if responses
-            Adapters::Test.new(responses: responses.map { |r| normalize_test_response(r) })
-          else
-            Adapters::Test.new(response: normalize_test_response(response))
-          end
-        end
-
-        def normalize_test_response(value)
-          value
-        end
+        include Concerns::StubHelpers
       end
     end
   end

data/lib/ruby_llm/contract/step/base.rb

@@ -21,19 +21,30 @@ module RubyLLM
                              context: context).results.first
           end
 
-          def estimate_cost(input:, model: nil)
+          def estimate_cost(input:, model: nil, attachment: nil)
             model_name = estimated_model_name(model)
-            model_info = CostCalculator.send(:find_model, model_name)
+            model_info = CostCalculator.find_model(model_name)
             return nil unless model_info
 
-            input_tokens = TokenEstimator.estimate(build_messages(input))
+            text_tokens = TokenEstimator.estimate(build_messages(input))
+            attachment_tokens, attachment_error = resolve_attachment_tokens(attachment)
+            return nil if attachment_error
+
+            input_tokens = text_tokens + attachment_tokens
+            # NOTE: attachment tokens add to input only, not output. Vision-
+            # heavy outputs (long image descriptions) may exceed
+            # `output_tokens_estimate` — this method is a floor for budget
+            # planning, not a precise predictor. See multimodal_input.md.
             output_tokens = max_output || DEFAULT_OUTPUT_TOKENS
 
             {
               model: model_name,
               input_tokens: input_tokens,
               output_tokens_estimate: output_tokens,
-              estimated_cost: estimated_cost_for(model_info, input_tokens, output_tokens)
+              estimated_cost: CostCalculator.calculate(
+                model_name: model_name,
+                usage: { input_tokens: input_tokens, output_tokens: output_tokens }
+              )
             }
           end
 
@@ -49,7 +60,9 @@ module RubyLLM
             end
           end
 
-          def recommend(eval_name, candidates:, min_score: 0.95, min_first_try_pass_rate: 0.8, context: {})
+          def recommend(eval_name, candidates:, context: {},
+                        min_score: Eval::DEFAULT_MIN_SCORE,
+                        min_first_try_pass_rate: Eval::DEFAULT_MIN_FIRST_TRY_PASS_RATE)
             comparison = compare_models(eval_name, candidates: candidates, context: context)
             Eval::Recommender.new(
               comparison: comparison,
@@ -59,7 +72,9 @@ module RubyLLM
             ).recommend
           end
 
-          def optimize_retry_policy(candidates:, context: {}, min_score: 0.95, runs: 1, production_mode: nil)
+          def optimize_retry_policy(candidates:, context: {},
+                                    min_score: Eval::DEFAULT_MIN_SCORE,
+                                    runs: 1, production_mode: nil)
             Eval::RetryOptimizer.new(
               step: self,
               candidates: candidates,
@@ -71,7 +86,7 @@ module RubyLLM
           end
 
           KNOWN_CONTEXT_KEYS = %i[adapter model temperature max_tokens provider assume_model_exists
-                                  reasoning_effort retry_policy_override].freeze
+                                  reasoning_effort retry_policy_override attachment].freeze
 
           include Concerns::ContextHelpers
 
@@ -104,12 +119,23 @@ module RubyLLM
             model || (self.model if respond_to?(:model)) || RubyLLM::Contract.configuration.default_model
           end
 
-          def estimated_cost_for(model_info, input_tokens, output_tokens)
-            CostCalculator.send(
-              :compute_cost,
-              model_info,
-              { input_tokens: input_tokens, output_tokens: output_tokens }
-            )
+          # Returns [tokens, error?] where error is true when fail-closed should
+          # short-circuit the caller. Mirrors limit_checker.rb fail-closed policy
+          # so estimate_cost and runtime check_limits agree on the same input.
+          def resolve_attachment_tokens(attachment)
+            return [0, false] if attachment.nil?
+
+            estimate = attachment_token_estimate if respond_to?(:attachment_token_estimate)
+            return [estimate, false] unless estimate.nil?
+
+            mode = respond_to?(:on_unknown_attachment_size) ? on_unknown_attachment_size : :refuse
+            if mode == :warn
+              warn "[ruby_llm-contract] attachment present but attachment_token_estimate not " \
+                   "declared on #{name || self} — estimate_cost proceeds without attachment cost"
+              return [0, false]
+            end
+
+            [0, true]
           end
 
           def estimate_eval_cost_for_model(cases, model_name)
@@ -159,7 +185,7 @@ module RubyLLM
 
           def runtime_settings(context)
             policy = context.key?(:retry_policy_override) ? context[:retry_policy_override] : retry_policy
-            extra = context.slice(:provider, :assume_model_exists, :max_tokens, :reasoning_effort)
+            extra = context.slice(:provider, :assume_model_exists, :max_tokens, :reasoning_effort, :attachment)
 
             # Always pass the class-level `thinking` config to the adapter when
             # set, so fields like `budget` survive a per-call `reasoning_effort`
@@ -215,18 +241,9 @@ module RubyLLM
           # programmer bugs (NoMethodError, adapter-code ArgumentError) must propagate
           # instead of being silently masked as :input_error.
           def run_once(input, adapter:, model:, context_temperature: nil, extra_options: {})
-            effective_temp = context_temperature || temperature
             runner =
               begin
-                Runner.new(
-                  input_type: input_type, output_type: output_type,
-                  prompt_block: prompt, contract_definition: effective_contract,
-                  adapter: adapter, model: model, output_schema: output_schema,
-                  max_output: max_output, max_input: max_input, max_cost: max_cost,
-                  on_unknown_pricing: on_unknown_pricing,
-                  temperature: effective_temp, extra_options: extra_options,
-                  observers: class_observers
-                )
+                Runner.new(config: build_runner_config(adapter, model, context_temperature, extra_options))
               rescue ArgumentError => e
                 return Result.new(status: :input_error, raw_output: nil, parsed_output: nil,
                                   validation_errors: [e.message])
@@ -235,6 +252,21 @@ module RubyLLM
             runner.call(input)
           end
 
+          def build_runner_config(adapter, model, context_temperature, extra_options)
+            RunnerConfig.build(
+              input_type: input_type, output_type: output_type,
+              prompt_block: prompt, contract_definition: effective_contract,
+              adapter: adapter, model: model, output_schema: output_schema,
+              max_output: max_output, max_input: max_input, max_cost: max_cost,
+              on_unknown_pricing: on_unknown_pricing,
+              attachment_token_estimate: attachment_token_estimate,
+              on_unknown_attachment_size: on_unknown_attachment_size,
+              temperature: context_temperature || temperature,
+              extra_options: extra_options,
+              observers: class_observers
+            )
+          end
+
           def log_result(result)
             logger = RubyLLM::Contract.configuration.logger
             return unless logger