squishling 0.2.0 → 0.4.0

data/lib/squishling/harness.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Squishling
+  # How a squished call uses its escalation. Declared as a type, or a Hash with the type and its options:
+  #   harness: :squishsum
+  #   harness: { type: :judged_squishsum, judge: "claude-opus-5-5", compare: ->(a, b, **) { a.team == b.team } }
+  # :escalation (the default) tries each attempt in order until an output passes. :squishsum asks the first
+  # escalation step twice, concurrently, and accepts the output only when both samples agree. :judged_squishsum
+  # hands disagreeing samples to a judge that picks one or rejects both. Your own types are registered with
+  # Squishling.register_harness (see Harness::Base).
+  class Harness
+    DEFAULT_JUDGE_INSTRUCTIONS = <<~TEXT.strip
+      You are an impartial judge. Two independent attempts at the same operation returned different outputs,
+      candidate "a" and candidate "b". Both already match the required output format, so judge only their
+      content. You are given the operation's purpose and its input. Decide which candidate correctly and
+      faithfully carries out the purpose for this input. If both do (for example, they differ only in
+      wording), choose either one, preferring "a". Choose "neither" only when both are wrong or you can't tell
+      whether either is correct. Never combine them or invent a third answer.
+    TEXT
+
+    # The type's validated options (frozen), and the Base subclass that runs it.
+    attr_reader :type, :klass, :options
+
+    class << self
+      def normalize(value, label)
+        case value
+        when Symbol, String then build(type_for(value, label), {}, label)
+        when Hash then from_hash(value, label)
+        else
+          raise ConfigurationError, "#{label}: harness must be one of #{types.join(', ')} or a Hash with type:, " \
+                                    "got #{value.inspect}"
+        end
+      end
+
+      # Every registered type: the built-ins, then your own in registration order.
+      def types
+        Registry.names
+      end
+
+      private
+
+      def from_hash(hash, label)
+        hash = hash.transform_keys(&:to_sym)
+        raise ConfigurationError, "#{label}: a harness Hash needs type:" unless hash.key?(:type)
+
+        build(type_for(hash[:type], label), hash.except(:type), label)
+      end
+
+      def build(type, options, label)
+        klass = Registry.fetch(type)
+        new(type:, klass:, options: klass.normalize_options(options, label))
+      end
+
+      def type_for(value, label)
+        type = value.to_s.to_sym if value.is_a?(Symbol) || value.is_a?(String)
+        return type if Registry.fetch(type)
+
+        raise ConfigurationError, "#{label}: unknown harness #{value.inspect} (use #{types.join(', ')})"
+      end
+    end
+
+    def initialize(type:, klass: Registry.fetch(type), options: {})
+      @type = type
+      @klass = klass
+      @options = options.freeze
+      freeze
+    end
+
+    DEFAULT = new(type: :escalation)
+
+    def squishsum?
+      %i[squishsum judged_squishsum].include?(type)
+    end
+
+    def judged?
+      type == :judged_squishsum
+    end
+  end
+end

data/lib/squishling/invoker.rb

@@ -8,119 +8,45 @@ module Squishling
       present, holds additional state about the caller. Respond only with JSON matching the required schema.
     NOTE
 
+    # The most of a rejected output that is forwarded to the next escalation step.
+    MAX_FORWARDED_CHARS = 4_000
+
+    # One escalation's request: the system prompt, the output Schema, and the first user message. The judge's
+    # role (:judge) skips squish_validate and names the judge in errors and logs.
+    Prompt = Data.define(:instructions, :schema, :input, :role)
+
     def initialize(definition, receiver, inputs)
       @definition = definition
       @receiver = receiver
       @inputs = inputs
     end
 
-    # Makes each attempt of the escalation (Definition#escalation_path) until an output passes the schema and
-    # the squish_validate check. Consecutive attempts on the same step continue the same conversation, so the
-    # model sees what it got wrong; a new step starts a fresh chat told about the last rejected output.
+    # Runs the harness (Definition#harness) over a Run of the call: the harness decides how the escalation is
+    # used, and what it returns is checked against the output schema's type.
     def call
-      @instructions = @definition.instructions(@receiver)
-      @schema = @definition.schema
-      raise ConfigurationError, "#{@definition.label} has no instructions" if @instructions.nil? || @instructions.empty?
-      raise ConfigurationError, "#{@definition.label} has no output_schema" unless @schema
-
-      path = @definition.escalation_path
-      input = JSON.generate(payload)
-      chat = nil
-      rejected = nil # [raw content, errors] of the last invalid output
-      models = []
-
-      path.each.with_index(1) do |step, attempt|
-        models << step.model
-        last = attempt == path.size
-        if chat && step == path[attempt - 2]
-          message = retry_message(rejected.last)
-        else
-          chat = start_chat(step)
-          message = rejected ? escalation_message(input, *rejected) : input
-        end
-
-        begin
-          response = ask(chat, message, step)
-        rescue LLMError => e
-          raise if last
-
-          # The failed request may be half-recorded in this chat, so the next attempt starts a fresh one.
-          chat = nil
-          log_failure(attempt, path, e.message)
-          next
-        end
-
-        result, errors = check(response.content)
-        return result if errors.empty?
+      purpose = @definition.purpose(@receiver)
+      schema = @definition.schema
+      raise ConfigurationError, "#{@definition.label} has no purpose" if purpose.nil? || purpose.empty?
+      raise ConfigurationError, "#{@definition.label} has no output_schema" unless schema
 
-        raise InvalidOutputError.new(errors:, raw: response.content, attempts: attempt, models:) if last
-
-        rejected = [response.content, errors]
-        log_failure(attempt, path, errors.join("; "))
-      end
+      harness = @definition.harness
+      body = payload
+      prompt = Prompt.new(instructions: "#{purpose}\n\n#{INPUT_NOTE}", schema:,
+        input: JSON.generate(body), role: nil)
+      run = Run.new(definition: @definition, receiver: @receiver, inputs: @inputs, prompt:, payload: body, purpose:,
+        harness:)
+      accepted(harness.klass.new.call(run, **harness.options), run, harness)
     end
 
     private
 
-    def start_chat(step)
-      chat = build_chat(step)
-      chat.with_instructions("#{@instructions}\n\n#{INPUT_NOTE}")
-      chat.with_schema(@schema.llm_schema)
-      apply_params(chat, step.params)
-      chat
-    end
-
-    def build_chat(step)
-      RubyLLM.chat(**chat_options(step))
-    rescue RubyLLM::ModelNotFoundError, RubyLLM::ConfigurationError => e
-      raise ConfigurationError, "#{@definition.label}: #{e.message}"
-    end
+    # What a harness returns must be a result its run produced: validated output, typed like the deterministic
+    # path's. One built by hand, or a hash, would skip the schema and squish_validate.
+    def accepted(value, run, harness)
+      return value if run.produced?(value)
 
-    # RubyLLM validates some settings locally (e.g. an impossible thinking budget for the model)
-    # and raises ArgumentError before any request is sent.
-    def apply_params(chat, params)
-      Params.apply(chat, params)
-    rescue ArgumentError => e
-      raise ConfigurationError, "#{@definition.label}: invalid params #{params.inspect} (#{e.message})"
-    end
-
-    # Transient HTTP failures are already retried by RubyLLM (RubyLLM.config.max_retries); anything
-    # that still fails is surfaced as an LLMError, which moves on to the next attempt of the
-    # escalation (or propagates from the last one). A 400 means the request we built is invalid (an
-    # unsupported param, a schema the provider rejects), so it's a setup mistake: escalating or a
-    # fallback would otherwise hide it on every call.
-    def ask(chat, message, step)
-      chat.ask(message)
-    rescue RubyLLM::ConfigurationError, RubyLLM::UnauthorizedError, RubyLLM::ForbiddenError => e
-      raise ConfigurationError, "#{@definition.label}: #{e.class}: #{e.message}"
-    rescue RubyLLM::BadRequestError => e
-      raise ConfigurationError,
-        "#{@definition.label}: the provider rejected the request (#{e.message})#{params_hint(step.params)}"
-    rescue RubyLLM::Error, Faraday::Error => e
-      raise LLMError, "#{@definition.label}: #{e.class}: #{e.message}"
-    end
-
-    # Models missing from RubyLLM's registry (e.g. newly released ones) are only usable when a
-    # provider is named, so RubyLLM is told to assume they exist.
-    def chat_options(step)
-      model = step.model
-      provider = step.provider
-      options = { model:, provider: }.compact
-      options[:assume_model_exists] = true if model && provider && !known_model?(model, provider)
-      options
-    end
-
-    def known_model?(model, provider)
-      RubyLLM.models.find(model, provider:)
-      true
-    rescue RubyLLM::ModelNotFoundError
-      false
-    end
-
-    def params_hint(params)
-      return "" if params.empty?
-
-      ". Check params #{params.inspect}; reasoning models often reject sampling params such as temperature and top_p"
+      raise ConfigurationError, "#{@definition.label}: the #{harness.type} harness must return a result from " \
+                                "its run (escalate, samples, or a judge's pick), got #{value.class}"
     end
 
     # Only the arguments, the declared squish_context names, and a squish! call's context leave the process.
@@ -153,93 +79,5 @@ module Squishling
 
       @receiver.instance_variable_get(:"@#{name}")
     end
-
-    # RubyLLM parses structured output itself and leaves the raw string when that fails, so
-    # content is a Hash on success, or a String/nil when the model refused, was cut off, or
-    # ignored the schema.
-    def parse(content)
-      return [nil, ["response was empty"]] if content.nil? || (content.is_a?(String) && content.strip.empty?)
-      return [content, []] unless content.is_a?(String)
-
-      [JSON.parse(strip_code_fence(content)), []]
-    rescue JSON::ParserError => e
-      [nil, ["response was not valid JSON (#{e.message.lines.first&.strip})"]]
-    end
-
-    # Models without native structured output sometimes wrap JSON in a markdown code fence.
-    def strip_code_fence(text)
-      text[/\A\s*```(?:json)?\s*\n(.*?)\n\s*```\s*\z/m, 1] || text
-    end
-
-    # Parses and validates a response: [typed result, []] when it passes, [nil or result, errors] otherwise.
-    def check(content)
-      data, errors = parse(content)
-      errors = @schema.validate(data) if errors.empty?
-      return [nil, errors] if errors.any?
-
-      result = @schema.build(data, squished: true)
-      [result, validator_errors(result)]
-    end
-
-    # Runs the squish_validate check, if any. Exceptions it raises are the user's own and propagate as-is.
-    def validator_errors(result)
-      validator = @definition.validator
-      return [] unless validator
-
-      value = @receiver.instance_exec(result, **@inputs, &validator)
-      case value
-      when nil, true then []
-      when false then ["the output was rejected by squish_validate"]
-      when String, Array then Array(value).flatten.compact.map(&:to_s).reject { |message| message.strip.empty? }
-      else
-        return contract_errors(value) if value.respond_to?(:success?) && value.respond_to?(:errors)
-
-        raise ConfigurationError, "#{@definition.label}: squish_validate must return nil, true, false, a String, " \
-                                  "an Array of Strings, or a validation result, got #{value.class}"
-      end
-    end
-
-    # A dry-validation style result: errors.to_h is { key => ["message", ...] }, nested for nested keys.
-    def contract_errors(value)
-      return [] if value.success?
-
-      errors = value.errors
-      errors = errors.to_h if !errors.is_a?(Array) && errors.respond_to?(:to_h)
-      messages = errors.is_a?(Hash) ? flatten_messages(errors) : Array(errors).map(&:to_s)
-      messages.empty? ? ["the output was rejected by squish_validate"] : messages
-    end
-
-    def flatten_messages(errors, prefix = nil)
-      errors.flat_map do |key, value|
-        path = [prefix, key].compact.join(".")
-        next flatten_messages(value, path) if value.is_a?(Hash)
-
-        Array(value).map { |message| path.empty? ? message.to_s : "#{path} #{message}" }
-      end
-    end
-
-    def retry_message(errors)
-      "Your previous response was rejected:\n- #{errors.join("\n- ")}\nRespond again with corrected JSON only."
-    end
-
-    # The first message to a fresh chat after an earlier model's output was rejected.
-    def escalation_message(input, raw, errors)
-      previous = raw.is_a?(String) ? raw : JSON.generate(raw)
-      previous = "(an empty response)" if previous.strip.empty? || raw.nil?
-      "#{input}\n\nA previous attempt at this request returned:\n#{previous}\n" \
-        "It was rejected:\n- #{errors.join("\n- ")}\nRespond with corrected JSON only."
-    end
-
-    def log_failure(attempt, path, reason)
-      step = path[attempt - 1]
-      next_step = path[attempt]
-      action = next_step == step ? "retrying" : "escalating to #{model_name(next_step)}"
-      Squishling.config.logger&.warn("[Squishling] #{@definition.label} attempt #{attempt} of #{path.size} " \
-                                     "(#{model_name(step)}) failed, #{action}: #{reason}")
-    end
-
-    def model_name(step)
-      step.model || "RubyLLM default model"
-    end
   end
 end

data/lib/squishling/judge.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+module Squishling
+  # Picks between two disagreeing samples, or rejects both (see Run#judge). The judge is the given judge:
+  # step, or else the escalation's next stage after the first. A chat judge answers with a strict verdict
+  # schema; a :judgment judge (a System One decision model such as Jev) answers one choice question through
+  # RubyLLM.judge, and its pick counts only at or above min_confidence.
+  class Judge
+    # The judge kinds: :chat (a chat model answering VERDICT_SCHEMA) and :judgment (a System One decision model
+    # through RubyLLM.judge). A judge is an escalation step (see ModelPath) plus these keys: type: (default :chat)
+    # and, for :judgment, min_confidence: (the probability the chosen candidate needs).
+    TYPES = %i[chat judgment].freeze
+    STEP_KEYS = %i[type min_confidence].freeze
+    DEFAULT_MIN_CONFIDENCE = 0.8
+
+    VERDICT_SCHEMA = {
+      "type" => "object",
+      "properties" => {
+        "verdict" => { "type" => "string", "enum" => %w[a b neither] },
+        "reason" => { "type" => "string" }
+      },
+      "required" => %w[verdict reason],
+      "additionalProperties" => false
+    }.freeze
+
+    INPUT_NOTE = <<~NOTE
+      The input is a JSON object. "input" is what the operation received ("arguments" and, when present,
+      "context"); "output_schema" is the format both candidates follow; "candidates" holds "a" and "b".
+      Respond only with JSON matching the required schema: your verdict ("a", "b", or "neither") and a short reason.
+    NOTE
+
+    CHOICES = {
+      a: "Candidate a correctly carries out the purpose for this input (also when both do)",
+      b: "Candidate b correctly carries out the purpose for this input",
+      neither: "Neither candidate is clearly correct"
+    }.freeze
+
+    # What a judge decided: :a, :b, or :neither, with its reason. The detail is text Squishling wrote (nil for a
+    # chat judge, whose reason is model-written) and is safe for error messages.
+    Verdict = Data.define(:choice, :reason, :detail)
+
+    # judge: is a normalized judge step (see Harness::JudgedSquishsum) or nil for the escalation's next stage,
+    # instructions: a String, a Proc evaluated against the receiver, or nil for the default prompt.
+    def initialize(definition:, receiver:, client:, judge:, instructions:, path:, purpose:, payload:, escalate:,
+      log_failure:, guard:, harness_type:)
+      @definition = definition
+      @receiver = receiver
+      @client = client
+      @judge = judge
+      @instructions = instructions
+      check!
+      @guard = guard
+      @harness_type = harness_type
+      @purpose = purpose
+      @payload = payload
+      @escalate = escalate
+      @log_failure = log_failure
+      @steps = resolve_steps(path)
+    end
+
+    # The judge's first attempt, for logs and DisagreementError#models.
+    def step
+      @steps.first
+    end
+
+    # The Verdict for the two candidates: :a or :b for the chosen one, or :neither.
+    def verdict(first, second)
+      @guard.call
+      candidates = { a: Schema.jsonify(first.to_h), b: Schema.jsonify(second.to_h) }
+      judgment? ? judgment(candidates) : chat(candidates)
+    end
+
+    private
+
+    # What Run#judge was given. A harness declaration is checked earlier, with its own messages.
+    def check!
+      label = "#{@definition.label}: judge"
+      if @judge
+        type = @judge.fetch(:type)
+        unless TYPES.include?(type)
+          raise ConfigurationError, "#{label} type: must be one of #{TYPES.join(', ')}, got #{type.inspect}"
+        end
+
+        if type == :chat && @judge.key?(:min_confidence)
+          raise ConfigurationError, "#{label} min_confidence: only applies to type: :judgment judges"
+        end
+
+        minimum = @judge.fetch(:min_confidence, DEFAULT_MIN_CONFIDENCE)
+        unless minimum.is_a?(Numeric) && minimum.between?(0, 1)
+          raise ConfigurationError, "#{label} min_confidence: must be a number from 0 to 1, got #{minimum.inspect}"
+        end
+      end
+      return if @instructions.nil? || @instructions.is_a?(Proc) || non_blank_string?(@instructions)
+
+      raise ConfigurationError, "#{label} instructions must be a non-blank String or a Proc"
+    end
+
+    def non_blank_string?(value)
+      value.is_a?(String) && !value.strip.empty?
+    end
+
+    def judgment?
+      @judge&.fetch(:type) == :judgment
+    end
+
+    # A declared judge brings its own model and provider. A chat judge gets the method's generation params
+    # under its own, like any escalation step; a judgment judge sends only its own params, as provider options.
+    def resolve_steps(path)
+      if (judge = @judge)
+        params = judge[:type] == :judgment ? {} : @definition.params
+        return ModelPath.steps([judge], provider: nil, params:)
+      end
+
+      rest = path.drop_while { |step| step == path.first }
+      if rest.empty?
+        raise ConfigurationError, "#{@definition.label}: the #{@harness_type} harness needs a judge: or a second " \
+                                  "escalation step to judge with"
+      end
+
+      rest.take_while { |step| step == rest.first }
+    end
+
+    def chat(candidates)
+      prompt = Invoker::Prompt.new(
+        instructions: "#{judge_instructions}\n\nThe operation's purpose:\n<purpose>\n#{@purpose}\n" \
+                      "</purpose>\n\n#{INPUT_NOTE}",
+        schema: Schema.for(VERDICT_SCHEMA), input: JSON.generate(state(candidates)), role: :judge
+      )
+      result = @escalate.call(@steps, prompt)
+      Verdict.new(choice: result[:verdict].to_sym, reason: result[:reason], detail: nil)
+    end
+
+    def judgment(candidates)
+      input = { purpose: @purpose, **state(candidates) }
+      questions = { winner: { type: :choice, instructions: judge_instructions, options: CHOICES } }
+      answer = judge_with_retries(input, questions)[:winner]
+      unless answer.respond_to?(:choice) && answer.respond_to?(:probabilities)
+        raise InvalidOutputError.new(errors: ["the judgment didn't answer the winner question"], source: "Judge")
+      end
+
+      interpret(answer.choice.to_s.to_sym, answer.probabilities)
+    end
+
+    # The threshold is checked on the exact probability; it's rounded only for the reason text.
+    def interpret(choice, probabilities)
+      probability = probabilities.to_h.find { |key, _| key.to_s == choice.to_s }&.last.to_f
+      minimum = @judge.fetch(:min_confidence, DEFAULT_MIN_CONFIDENCE)
+      shown = probability.round(3)
+      picked = %i[a b].include?(choice)
+      if picked && probability >= minimum
+        text = "chosen with probability #{shown}"
+        return Verdict.new(choice:, reason: text, detail: text)
+      end
+
+      text =
+        if choice == :neither then "the judge chose neither (probability #{shown})"
+        elsif picked then "the judge chose #{choice} with probability #{shown}, below min_confidence #{minimum}"
+        else "the judge returned an unrecognized choice (probability #{shown})"
+        end
+      Verdict.new(choice: :neither, reason: text, detail: text)
+    end
+
+    # A judgment always matches its questions, so only a failed request (LLMError) moves on to the next attempt.
+    def judge_with_retries(input, questions)
+      @steps.each.with_index(1) do |step, attempt|
+        return @client.observed_judgment(input, questions:, step:, attempt:, attempts: @steps.size)
+      rescue LLMError => e
+        raise if attempt == @steps.size
+
+        @log_failure.call(:judge, attempt, @steps, e.message)
+      end
+    end
+
+    def state(candidates)
+      { input: @payload, output_schema: @definition.schema.llm_schema["schema"], candidates: }
+    end
+
+    def judge_instructions
+      value = @instructions || Harness::DEFAULT_JUDGE_INSTRUCTIONS
+      value = @receiver.instance_exec(&value) if value.is_a?(Proc)
+      return value if value.is_a?(String) && !value.strip.empty?
+
+      raise ConfigurationError, "#{@definition.label}: the judge_instructions proc must return a non-blank String"
+    end
+  end
+end

data/lib/squishling/llm_client.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+module Squishling
+  # RubyLLM requests for one squished method, with RubyLLM's errors mapped onto Squishling's taxonomy.
+  class LLMClient
+    # Where RubyLLM's own code lives, to tell its ArgumentErrors from the developer's (see #judge).
+    RUBY_LLM_LIB = File.dirname(RubyLLM.method(:judge).source_location.first)
+    private_constant :RUBY_LLM_LIB
+
+    # Runs each job on its own thread and returns, in order, each job's [value, exception]. Only provider
+    # requests run here: parsing, validation, and the developer's callbacks stay on the caller's thread. Fiber
+    # storage (RubyLLM's usage owner) is inherited by the threads, and RubyLLM's instrumentation context is
+    # carried over the way RubyLLM's own concurrent tool calls do it. Unlike those, the threads don't enter the
+    # Rails executor: the caller already holds it, and a second share can deadlock against a pending code reload.
+    # If the caller is interrupted while waiting (a timeout, Thread#raise), the requests still running are
+    # stopped rather than left running unobserved.
+    def self.concurrently(jobs)
+      return jobs.map { |job| capture(job) } if jobs.size < 2
+
+      context = instrumentation_context
+      threads = jobs.map do |job|
+        thread = Thread.new { with_instrumentation_context(context) { capture(job) } }
+        thread.report_on_exception = false
+        thread
+      end
+      threads.map(&:value)
+    ensure
+      threads&.each { |thread| thread.kill.join if thread.alive? }
+    end
+
+    def self.capture(job)
+      [job.call, nil]
+    rescue Exception => e # rubocop:disable Lint/RescueException -- re-raised on the caller's thread
+      [nil, e]
+    end
+
+    def self.instrumentation_context
+      instrumentation = defined?(RubyLLM::Support::Instrumentation) && RubyLLM::Support::Instrumentation
+      return unless instrumentation.respond_to?(:current_workflow) && instrumentation.respond_to?(:capture_context)
+
+      [instrumentation.current_workflow, instrumentation.capture_context]
+    end
+
+    def self.with_instrumentation_context(context, &)
+      return yield unless context
+
+      instrumentation = RubyLLM::Support::Instrumentation
+      instrumentation.with_workflow(context.first) { instrumentation.with_context(context.last, &) }
+    end
+
+    private_class_method :capture, :instrumentation_context, :with_instrumentation_context
+
+    # purpose, harness (the harness type), and handed_off (squish! sent the call here) are only told to the observer.
+    def initialize(label, purpose:, harness:, handed_off:)
+      @label = label
+      @purpose = purpose
+      @harness = harness
+      @handed_off = handed_off
+    end
+
+    # A fresh chat on one escalation step, with the system prompt, the strict output schema, and its params.
+    def chat(step, instructions:, schema:)
+      chat = build_chat(step)
+      chat.with_instructions(instructions)
+      chat.with_schema(schema)
+      apply_params(chat, step.params)
+      chat
+    end
+
+    # Transient HTTP failures are already retried by RubyLLM (RubyLLM.config.max_retries); anything
+    # that still fails is surfaced as an LLMError, which moves on to the next attempt of the
+    # escalation (or propagates from the last one). A 400 means the request we built is invalid (an
+    # unsupported param, a schema the provider rejects), so it's a setup mistake: escalating or a
+    # fallback would otherwise hide it on every call.
+    # The request runs inside the configured observer (see Observer), on this thread: for samples, the worker's.
+    # role is nil for an escalation attempt, :sample (with its name) or :judge; attempt counts in attempts.
+    def ask(chat, message, step, attempt:, attempts:, role: nil, sample: nil)
+      call = call_info(step, :chat, role:, sample:, attempt:, attempts:)
+      Observer.around(call) { request(step) { chat.ask(message) } }
+    end
+
+    # A System One judgment (RubyLLM.judge) on one step. The step's params are sent as provider options.
+    # RubyLLM raises ArgumentError for a judgment it can't build (e.g. a model on a provider that takes none, or
+    # a provider option its protocol reserves), and a bare RubyLLM::Error without an HTTP response for a provider
+    # that doesn't support judgments at all. An ArgumentError from anywhere else (an instrumentation subscriber
+    # runs inside the call) is the developer's own and propagates as-is.
+    def judge(input, questions:, step:)
+      request(step) do
+        RubyLLM.judge(input, questions:, provider_options: step.params, **model_options(step))
+      rescue RubyLLM::ModelNotFoundError => e
+        raise ConfigurationError, "#{@label}: #{e.message}"
+      rescue ArgumentError => e
+        raise unless raised_by_ruby_llm?(e)
+
+        raise ConfigurationError, "#{@label}: #{e.message}"
+      rescue RubyLLM::Error => e
+        raise unless e.instance_of?(RubyLLM::Error) && e.response.nil?
+
+        raise ConfigurationError, "#{@label}: #{e.message}"
+      end
+    end
+
+    # #judge inside the configured observer (see Observer), attempt counting in attempts.
+    def observed_judgment(input, questions:, step:, attempt:, attempts:)
+      call = call_info(step, :judgment, role: :judge, attempt:, attempts:)
+      Observer.around(call) { judge(input, questions:, step:) }
+    end
+
+    private
+
+    def call_info(step, request, attempt:, attempts:, role: nil, sample: nil)
+      Observer::Call.new(label: @label, purpose: @purpose, harness: @harness, handed_off: @handed_off, role:, sample:,
+        attempt:, attempts:, model: step.model, provider: step.provider, request:)
+    end
+
+    def raised_by_ruby_llm?(error)
+      error.backtrace&.first&.start_with?("#{RUBY_LLM_LIB}/") || false
+    end
+
+    def build_chat(step)
+      RubyLLM.chat(**model_options(step))
+    rescue RubyLLM::ModelNotFoundError, RubyLLM::ConfigurationError => e
+      raise ConfigurationError, "#{@label}: #{e.message}"
+    end
+
+    # RubyLLM validates some settings locally (e.g. an impossible thinking budget for the model)
+    # and raises ArgumentError before any request is sent.
+    def apply_params(chat, params)
+      Params.apply(chat, params)
+    rescue ArgumentError => e
+      raise ConfigurationError, "#{@label}: invalid params #{params.inspect} (#{e.message})"
+    end
+
+    # A provider's or transport's own message never goes into ours: a provider can echo request content or part of
+    # an API key in it, and ours reach logs and error trackers. The original stays as #cause for whoever
+    # chooses to read it. RubyLLM's own ConfigurationError is local (it names a missing setting), so it is kept.
+    def request(step)
+      yield
+    rescue RubyLLM::ConfigurationError => e
+      raise ConfigurationError, "#{@label}: #{e.class}: #{e.message}"
+    rescue RubyLLM::UnauthorizedError, RubyLLM::ForbiddenError => e
+      raise ConfigurationError, "#{@label}: #{describe_failure(e)}"
+    rescue RubyLLM::BadRequestError => e
+      raise ConfigurationError,
+        "#{@label}: the provider rejected the request (#{describe_failure(e)})#{params_hint(step.params)}"
+    rescue RubyLLM::Error, Faraday::Error => e
+      raise LLMError, "#{@label}: #{describe_failure(e)}"
+    end
+
+    # The error's class and HTTP status, when it has one, which is all of it that is ours to repeat.
+    def describe_failure(error)
+      response = error.response
+      status = response.is_a?(Hash) ? response[:status] : (response.status if response.respond_to?(:status))
+      status ? "#{error.class}, HTTP #{status}" : error.class.to_s
+    end
+
+    # Models missing from RubyLLM's registry (e.g. newly released ones) are only usable when a
+    # provider is named, so RubyLLM is told to assume they exist.
+    def model_options(step)
+      model = step.model
+      provider = step.provider
+      options = { model:, provider: }.compact
+      options[:assume_model_exists] = true if model && provider && !known_model?(model, provider)
+      options
+    end
+
+    def known_model?(model, provider)
+      RubyLLM.models.find(model, provider:)
+      true
+    rescue RubyLLM::ModelNotFoundError
+      false
+    end
+
+    def params_hint(params)
+      return "" if params.empty?
+
+      ". Check params #{params.inspect}; reasoning models often reject sampling params such as temperature and top_p"
+    end
+  end
+end