squishling 0.3.0 → 0.4.0

data/lib/squishling/invoker.rb

@@ -15,18 +15,14 @@ module Squishling
     # role (:judge) skips squish_validate and names the judge in errors and logs.
     Prompt = Data.define(:instructions, :schema, :input, :role)
 
-    # One of a sampling harness's two samples, between rounds: the steps it may attempt, in order.
-    Sample = Struct.new(:name, :steps, :chat, :rejected, :result)
-
     def initialize(definition, receiver, inputs)
       @definition = definition
       @receiver = receiver
       @inputs = inputs
-      @client = LLMClient.new(definition.label)
-      @output_check = OutputCheck.new(definition, receiver, inputs)
     end
 
-    # Runs the harness (Definition#harness) over the escalation (Definition#escalation_path).
+    # Runs the harness (Definition#harness) over a Run of the call: the harness decides how the escalation is
+    # used, and what it returns is checked against the output schema's type.
     def call
       purpose = @definition.purpose(@receiver)
       schema = @definition.schema
@@ -34,173 +30,23 @@ module Squishling
       raise ConfigurationError, "#{@definition.label} has no output_schema" unless schema
 
       harness = @definition.harness
-      path = @definition.escalation_path
       body = payload
       prompt = Prompt.new(instructions: "#{purpose}\n\n#{INPUT_NOTE}", schema:,
         input: JSON.generate(body), role: nil)
-      return escalate(path, prompt) unless harness.squishsum?
-
-      # The sample steps and the judge are resolved before any sample is requested, so a missing one fails
-      # before costing anything.
-      sample_steps, rest = sampling_plan(path, harness)
-      judge = harness.judged? && Judge.new(definition: @definition, receiver: @receiver, client: @client, harness:,
-        rest:, purpose:, payload: body, escalate: method(:escalate),
-        log_failure: method(:log_failure))
-      sample(sample_steps, prompt, harness, judge)
+      run = Run.new(definition: @definition, receiver: @receiver, inputs: @inputs, prompt:, payload: body, purpose:,
+        harness:)
+      accepted(harness.klass.new.call(run, **harness.options), run, harness)
     end
 
     private
 
-    # Splits the escalation into runs of consecutive identical steps (a step's attempts). A squishsum samples
-    # the first run twice; an ensemble samples the first run, then the second. Returns each sample's steps and
-    # the rest of the escalation, which is where a judge looks for its default step.
-    def sampling_plan(path, harness)
-      runs = path.chunk_while { |step, following| step == following }.to_a
-      if harness.ensemble? && runs.size < 2
-        raise ConfigurationError, "#{@definition.label}: the #{harness.type} harness needs a second escalation " \
-                                  "step to sample (adjacent identical steps count as one step's attempts)"
-      end
-
-      return [runs.first(2), runs.drop(2).flatten(1)] if harness.ensemble?
-
-      [[runs.first, runs.first], runs.drop(1).flatten(1)]
-    end
-
-    # Makes each attempt of the path until an output passes the schema and the squish_validate check.
-    # Consecutive attempts on the same step continue the same conversation, so the model sees what it got wrong;
-    # a new step starts a fresh chat told about the last rejected output (unless the step sets
-    # forward_rejected: false).
-    def escalate(path, prompt)
-      chat = nil
-      rejected = nil # [raw content, errors] of the last invalid output
-      models = []
-
-      path.each.with_index(1) do |step, attempt|
-        models << step.model
-        last = attempt == path.size
-        if chat && step == path[attempt - 2]
-          message = retry_message(rejected.last)
-        else
-          chat = start_chat(step, prompt)
-          message = rejected && step.forward_rejected ? escalation_message(prompt.input, *rejected) : prompt.input
-        end
-
-        begin
-          response = @client.ask(chat, message, step)
-        rescue LLMError => e
-          squawk(prompt, path, attempt, nil, e)
-          raise if last
-
-          # The failed request may be half-recorded in this chat, so the next attempt starts a fresh one.
-          chat = nil
-          log_failure(prompt.role, attempt, path, e.message)
-          next
-        end
-
-        result, errors = @output_check.call(response.content, prompt)
-        if errors.empty?
-          squawk(prompt, path, attempt, response, nil)
-          return result
-        end
-
-        error = InvalidOutputError.new(errors:, raw: response.content, attempts: attempt, models: models.dup,
-          source: source(prompt.role))
-        squawk(prompt, path, attempt, response, error)
-        raise error if last
-
-        rejected = [response.content, errors]
-        log_failure(prompt.role, attempt, path, errors.join("; "))
-      end
-    end
-
-    # Two samples, each on its own steps and retried per their attempts the same way escalate retries.
-    # Each round sends the pending samples' requests concurrently; everything else runs on this thread.
-    def sample(sample_steps, prompt, harness, judge)
-      samples = %w[a b].zip(sample_steps).map { |name, steps| Sample.new(name, steps) }
-
-      1.upto(sample_steps.map(&:size).max) do |attempt|
-        pending = samples.reject(&:result)
-        break if pending.empty?
-
-        jobs = pending.map do |sample|
-          step = sample.steps[attempt - 1]
-          message = sample_message(sample, step, prompt)
-          -> { @client.ask(sample.chat, message, step) }
-        end
-        outcomes = LLMClient.concurrently(jobs)
-        # A setup mistake (or anything that isn't a failed request) ends the call, whichever sample hit it.
-        fatal = outcomes.map(&:last).find { |error| error && !error.is_a?(LLMError) }
-        raise fatal if fatal
-
-        pending.zip(outcomes).each { |sample, outcome| record(sample, outcome, prompt, attempt) }
-      end
-
-      settle(samples, harness, judge)
-    end
-
-    # The next message for a sample: a retry in its chat after invalid output, otherwise a fresh chat.
-    def sample_message(sample, step, prompt)
-      return retry_message(sample.rejected.last) if sample.chat
-
-      sample.chat = start_chat(step, prompt)
-      sample.rejected && step.forward_rejected ? escalation_message(prompt.input, *sample.rejected) : prompt.input
-    end
-
-    def record(sample, (response, error), prompt, attempt)
-      steps = sample.steps
-      last = attempt == steps.size
-      role = "sample #{sample.name}"
-      if error
-        squawk(prompt, steps, attempt, nil, error)
-        raise error if last
-
-        sample.chat = nil
-        return log_failure(role, attempt, steps, error.message)
-      end
-
-      result, errors = @output_check.call(response.content, prompt)
-      if errors.empty?
-        squawk(prompt, steps, attempt, response, nil)
-        return sample.result = result
-      end
-
-      models = steps.first(attempt).map(&:model)
-      invalid = InvalidOutputError.new(errors:, raw: response.content, attempts: attempt, models:)
-      squawk(prompt, steps, attempt, response, invalid)
-      raise invalid if last
+    # What a harness returns must be a result its run produced: validated output, typed like the deterministic
+    # path's. One built by hand, or a hash, would skip the schema and squish_validate.
+    def accepted(value, run, harness)
+      return value if run.produced?(value)
 
-      sample.rejected = [response.content, errors]
-      log_failure(role, attempt, steps, errors.join("; "))
-    end
-
-    # Agreeing samples are accepted; otherwise the judge, if any, picks one or the call fails.
-    def settle(samples, harness, judge)
-      first, second = samples.map(&:result)
-      return first if agree?(first, second, harness.compare)
-
-      models = samples.map { |sample| sample.steps.first.model }
-      unless judge
-        log_warning("samples disagreed")
-        raise DisagreementError.new(candidates: [first, second], models:)
-      end
-
-      log_warning("samples disagreed, asking the judge (#{judge.step.display_name})")
-      choice, reason, detail = judge.verdict(first, second)
-      return { a: first, b: second }.fetch(choice) unless choice == :neither
-
-      raise DisagreementError.new(candidates: [first, second], verdict: :neither, reason:, detail:,
-        models: models + [judge.step.model])
-    end
-
-    # compare: is the developer's own code, so its exceptions propagate unwrapped.
-    def agree?(first, second, compare)
-      return Schema.jsonify(first.to_h) == Schema.jsonify(second.to_h) unless compare
-
-      @receiver.instance_exec(first, second, **@inputs, &compare) ? true : false
-    end
-
-    def start_chat(step, prompt)
-      @client.chat(step, instructions: prompt.instructions, schema: prompt.schema.llm_schema)
+      raise ConfigurationError, "#{@definition.label}: the #{harness.type} harness must return a result from " \
+                                "its run (escalate, samples, or a judge's pick), got #{value.class}"
     end
 
     # Only the arguments, the declared squish_context names, and a squish! call's context leave the process.
@@ -233,68 +79,5 @@ module Squishling
 
       @receiver.instance_variable_get(:"@#{name}")
     end
-
-    def retry_message(errors)
-      "Your previous response was rejected:\n- #{errors.join("\n- ")}\nRespond again with corrected JSON only."
-    end
-
-    # The first message to a fresh chat after an earlier model's output was rejected. The rejected output is
-    # model-generated and may cross providers, so it is capped at MAX_FORWARDED_CHARS.
-    def escalation_message(input, raw, errors)
-      previous = rejected_text(raw)
-      previous = "(an empty response)" if previous.strip.empty? || raw.nil?
-      if previous.length > MAX_FORWARDED_CHARS
-        omitted = previous.length - MAX_FORWARDED_CHARS
-        previous = "#{previous[0, MAX_FORWARDED_CHARS]}... [truncated, #{omitted} more characters]"
-      end
-      "#{input}\n\nA previous attempt at this request returned:\n#{previous}\n" \
-        "It was rejected:\n- #{errors.join("\n- ")}\nRespond with corrected JSON only."
-    end
-
-    # The rejected output as text. Output that was rejected for being unrepresentable in JSON (Infinity, invalid
-    # UTF-8) can't be serialized or measured as is, so it is scrubbed or replaced rather than raising.
-    def rejected_text(raw)
-      return utf8(raw).scrub("?") if raw.is_a?(String)
-
-      JSON.generate(raw)
-    rescue JSON::JSONError
-      "(a response JSON can't represent)"
-    end
-
-    # A binary-tagged string counts every byte as valid, so it is read as UTF-8 before scrubbing.
-    def utf8(text)
-      text.encoding == Encoding::BINARY ? text.dup.force_encoding(Encoding::UTF_8) : text
-    end
-
-    # Runs the observability hook, if any, with this attempt's raw output (nil when the call itself failed),
-    # the error that ended it (nil when it was accepted), and what was asked.
-    def squawk(prompt, path, attempt, response, error)
-      hook = @definition.squawk
-      return unless hook
-
-      step = path[attempt - 1]
-      metadata = {
-        label: @definition.label, attempt:, attempts: path.size, final: attempt == path.size,
-        model: response&.model || step.model, provider: step.provider, params: step.params, input: prompt.input,
-        usage: response&.tokens&.to_h
-      }
-      Squawk.call(hook, output: response&.content, metadata:, error:)
-    end
-
-    def source(role)
-      role == :judge ? "Judge" : "LLM"
-    end
-
-    def log_failure(role, attempt, path, reason)
-      step = path[attempt - 1]
-      next_step = path[attempt]
-      action = next_step == step ? "retrying" : "escalating to #{next_step.display_name}"
-      who = role ? "#{role} attempt" : "attempt"
-      log_warning("#{who} #{attempt} of #{path.size} (#{step.display_name}) failed, #{action}: #{reason}")
-    end
-
-    def log_warning(message)
-      Squishling.config.logger&.warn("[Squishling] #{@definition.label} #{message}")
-    end
   end
 end

data/lib/squishling/judge.rb

@@ -1,12 +1,18 @@
 # frozen_string_literal: true
 
 module Squishling
-  # Picks between a judged harness's two disagreeing samples (judged_squishsum or judged_ensemble), or rejects
-  # both. The judge is the harness's judge: step, or else the escalation's next step after the ones the samples
-  # ran on. A chat judge answers with a strict verdict schema; a :judgment judge (a System One decision model such
-  # as Jev) answers one choice question through RubyLLM.judge, and its pick counts only at or above
-  # min_confidence.
+  # Picks between two disagreeing samples, or rejects both (see Run#judge). The judge is the given judge:
+  # step, or else the escalation's next stage after the first. A chat judge answers with a strict verdict
+  # schema; a :judgment judge (a System One decision model such as Jev) answers one choice question through
+  # RubyLLM.judge, and its pick counts only at or above min_confidence.
   class Judge
+    # The judge kinds: :chat (a chat model answering VERDICT_SCHEMA) and :judgment (a System One decision model
+    # through RubyLLM.judge). A judge is an escalation step (see ModelPath) plus these keys: type: (default :chat)
+    # and, for :judgment, min_confidence: (the probability the chosen candidate needs).
+    TYPES = %i[chat judgment].freeze
+    STEP_KEYS = %i[type min_confidence].freeze
+    DEFAULT_MIN_CONFIDENCE = 0.8
+
     VERDICT_SCHEMA = {
       "type" => "object",
       "properties" => {
@@ -29,16 +35,27 @@ module Squishling
       neither: "Neither candidate is clearly correct"
     }.freeze
 
-    def initialize(definition:, receiver:, client:, harness:, rest:, purpose:, payload:, escalate:, log_failure:)
+    # What a judge decided: :a, :b, or :neither, with its reason. The detail is text Squishling wrote (nil for a
+    # chat judge, whose reason is model-written) and is safe for error messages.
+    Verdict = Data.define(:choice, :reason, :detail)
+
+    # judge: is a normalized judge step (see Harness::JudgedSquishsum) or nil for the escalation's next stage,
+    # instructions: a String, a Proc evaluated against the receiver, or nil for the default prompt.
+    def initialize(definition:, receiver:, client:, judge:, instructions:, path:, purpose:, payload:, escalate:,
+      log_failure:, guard:, harness_type:)
       @definition = definition
       @receiver = receiver
       @client = client
-      @harness = harness
+      @judge = judge
+      @instructions = instructions
+      check!
+      @guard = guard
+      @harness_type = harness_type
       @purpose = purpose
       @payload = payload
       @escalate = escalate
       @log_failure = log_failure
-      @steps = resolve_steps(rest)
+      @steps = resolve_steps(path)
     end
 
     # The judge's first attempt, for logs and DisagreementError#models.
@@ -46,31 +63,58 @@ module Squishling
       @steps.first
     end
 
-    # [:a | :b, reason, detail] for the chosen candidate, or [:neither, reason, detail]. The detail is text
-    # Squishling wrote (nil for a chat judge, whose reason is model-written) and is safe for error messages.
+    # The Verdict for the two candidates: :a or :b for the chosen one, or :neither.
     def verdict(first, second)
+      @guard.call
       candidates = { a: Schema.jsonify(first.to_h), b: Schema.jsonify(second.to_h) }
       judgment? ? judgment(candidates) : chat(candidates)
     end
 
     private
 
+    # What Run#judge was given. A harness declaration is checked earlier, with its own messages.
+    def check!
+      label = "#{@definition.label}: judge"
+      if @judge
+        type = @judge.fetch(:type)
+        unless TYPES.include?(type)
+          raise ConfigurationError, "#{label} type: must be one of #{TYPES.join(', ')}, got #{type.inspect}"
+        end
+
+        if type == :chat && @judge.key?(:min_confidence)
+          raise ConfigurationError, "#{label} min_confidence: only applies to type: :judgment judges"
+        end
+
+        minimum = @judge.fetch(:min_confidence, DEFAULT_MIN_CONFIDENCE)
+        unless minimum.is_a?(Numeric) && minimum.between?(0, 1)
+          raise ConfigurationError, "#{label} min_confidence: must be a number from 0 to 1, got #{minimum.inspect}"
+        end
+      end
+      return if @instructions.nil? || @instructions.is_a?(Proc) || non_blank_string?(@instructions)
+
+      raise ConfigurationError, "#{label} instructions must be a non-blank String or a Proc"
+    end
+
+    def non_blank_string?(value)
+      value.is_a?(String) && !value.strip.empty?
+    end
+
     def judgment?
-      @harness.judge&.fetch(:type) == :judgment
+      @judge&.fetch(:type) == :judgment
     end
 
     # A declared judge brings its own model and provider. A chat judge gets the method's generation params
     # under its own, like any escalation step; a judgment judge sends only its own params, as provider options.
-    def resolve_steps(rest)
-      if (judge = @harness.judge)
+    def resolve_steps(path)
+      if (judge = @judge)
         params = judge[:type] == :judgment ? {} : @definition.params
         return ModelPath.steps([judge], provider: nil, params:)
       end
 
+      rest = path.drop_while { |step| step == path.first }
       if rest.empty?
-        ordinal = @harness.ensemble? ? "third" : "second"
-        raise ConfigurationError, "#{@definition.label}: the #{@harness.type} harness needs a judge: or a " \
-                                  "#{ordinal} escalation step to judge with"
+        raise ConfigurationError, "#{@definition.label}: the #{@harness_type} harness needs a judge: or a second " \
+                                  "escalation step to judge with"
       end
 
       rest.take_while { |step| step == rest.first }
@@ -83,7 +127,7 @@ module Squishling
         schema: Schema.for(VERDICT_SCHEMA), input: JSON.generate(state(candidates)), role: :judge
       )
       result = @escalate.call(@steps, prompt)
-      [result[:verdict].to_sym, result[:reason], nil]
+      Verdict.new(choice: result[:verdict].to_sym, reason: result[:reason], detail: nil)
     end
 
     def judgment(candidates)
@@ -100,12 +144,12 @@ module Squishling
     # The threshold is checked on the exact probability; it's rounded only for the reason text.
     def interpret(choice, probabilities)
       probability = probabilities.to_h.find { |key, _| key.to_s == choice.to_s }&.last.to_f
-      minimum = @harness.judge[:min_confidence]
+      minimum = @judge.fetch(:min_confidence, DEFAULT_MIN_CONFIDENCE)
       shown = probability.round(3)
       picked = %i[a b].include?(choice)
       if picked && probability >= minimum
         text = "chosen with probability #{shown}"
-        return [choice, text, text]
+        return Verdict.new(choice:, reason: text, detail: text)
       end
 
       text =
@@ -113,13 +157,13 @@ module Squishling
         elsif picked then "the judge chose #{choice} with probability #{shown}, below min_confidence #{minimum}"
         else "the judge returned an unrecognized choice (probability #{shown})"
         end
-      [:neither, text, text]
+      Verdict.new(choice: :neither, reason: text, detail: text)
     end
 
     # A judgment always matches its questions, so only a failed request (LLMError) moves on to the next attempt.
     def judge_with_retries(input, questions)
       @steps.each.with_index(1) do |step, attempt|
-        return @client.judge(input, questions:, step:)
+        return @client.observed_judgment(input, questions:, step:, attempt:, attempts: @steps.size)
       rescue LLMError => e
         raise if attempt == @steps.size
 
@@ -132,7 +176,7 @@ module Squishling
     end
 
     def judge_instructions
-      value = @harness.judge_instructions || Harness::DEFAULT_JUDGE_INSTRUCTIONS
+      value = @instructions || Harness::DEFAULT_JUDGE_INSTRUCTIONS
       value = @receiver.instance_exec(&value) if value.is_a?(Proc)
       return value if value.is_a?(String) && !value.strip.empty?
 

data/lib/squishling/llm_client.rb

@@ -50,8 +50,12 @@ module Squishling
 
     private_class_method :capture, :instrumentation_context, :with_instrumentation_context
 
-    def initialize(label)
+    # purpose, harness (the harness type), and handed_off (squish! sent the call here) are only told to the observer.
+    def initialize(label, purpose:, harness:, handed_off:)
       @label = label
+      @purpose = purpose
+      @harness = harness
+      @handed_off = handed_off
     end
 
     # A fresh chat on one escalation step, with the system prompt, the strict output schema, and its params.
@@ -68,8 +72,11 @@ module Squishling
     # escalation (or propagates from the last one). A 400 means the request we built is invalid (an
     # unsupported param, a schema the provider rejects), so it's a setup mistake: escalating or a
     # fallback would otherwise hide it on every call.
-    def ask(chat, message, step)
-      request(step) { chat.ask(message) }
+    # The request runs inside the configured observer (see Observer), on this thread: for samples, the worker's.
+    # role is nil for an escalation attempt, :sample (with its name) or :judge; attempt counts in attempts.
+    def ask(chat, message, step, attempt:, attempts:, role: nil, sample: nil)
+      call = call_info(step, :chat, role:, sample:, attempt:, attempts:)
+      Observer.around(call) { request(step) { chat.ask(message) } }
     end
 
     # A System One judgment (RubyLLM.judge) on one step. The step's params are sent as provider options.
@@ -93,8 +100,19 @@ module Squishling
       end
     end
 
+    # #judge inside the configured observer (see Observer), attempt counting in attempts.
+    def observed_judgment(input, questions:, step:, attempt:, attempts:)
+      call = call_info(step, :judgment, role: :judge, attempt:, attempts:)
+      Observer.around(call) { judge(input, questions:, step:) }
+    end
+
     private
 
+    def call_info(step, request, attempt:, attempts:, role: nil, sample: nil)
+      Observer::Call.new(label: @label, purpose: @purpose, harness: @harness, handed_off: @handed_off, role:, sample:,
+        attempt:, attempts:, model: step.model, provider: step.provider, request:)
+    end
+
     def raised_by_ruby_llm?(error)
       error.backtrace&.first&.start_with?("#{RUBY_LLM_LIB}/") || false
     end
@@ -113,15 +131,27 @@ module Squishling
       raise ConfigurationError, "#{@label}: invalid params #{params.inspect} (#{e.message})"
     end
 
+    # A provider's or transport's own message never goes into ours: a provider can echo request content or part of
+    # an API key in it, and ours reach logs and error trackers. The original stays as #cause for whoever
+    # chooses to read it. RubyLLM's own ConfigurationError is local (it names a missing setting), so it is kept.
     def request(step)
       yield
-    rescue RubyLLM::ConfigurationError, RubyLLM::UnauthorizedError, RubyLLM::ForbiddenError => e
+    rescue RubyLLM::ConfigurationError => e
       raise ConfigurationError, "#{@label}: #{e.class}: #{e.message}"
+    rescue RubyLLM::UnauthorizedError, RubyLLM::ForbiddenError => e
+      raise ConfigurationError, "#{@label}: #{describe_failure(e)}"
     rescue RubyLLM::BadRequestError => e
       raise ConfigurationError,
-        "#{@label}: the provider rejected the request (#{e.message})#{params_hint(step.params)}"
+        "#{@label}: the provider rejected the request (#{describe_failure(e)})#{params_hint(step.params)}"
     rescue RubyLLM::Error, Faraday::Error => e
-      raise LLMError, "#{@label}: #{e.class}: #{e.message}"
+      raise LLMError, "#{@label}: #{describe_failure(e)}"
+    end
+
+    # The error's class and HTTP status, when it has one, which is all of it that is ours to repeat.
+    def describe_failure(error)
+      response = error.response
+      status = response.is_a?(Hash) ? response[:status] : (response.status if response.respond_to?(:status))
+      status ? "#{error.class}, HTTP #{status}" : error.class.to_s
     end
 
     # Models missing from RubyLLM's registry (e.g. newly released ones) are only usable when a

data/lib/squishling/model_path.rb

@@ -8,9 +8,11 @@ module Squishling
   #     "claude-sonnet-5-5",
   #     { model: "claude-opus-5-5", params: { thinking: { effort: :high } } }
   #   ]
-  # attempts: (default 1) retries a step; order: (all steps or none, unique, lowest first) makes the order
-  # explicit instead of positional. forward_rejected: false starts a step from the original input alone, without
-  # the previous step's rejected output.
+  # attempts: (default 1) retries a step; priority: (all steps or none, lowest first) makes the order explicit
+  # instead of positional. Steps sharing a priority are one tier: each call rotates which of them goes first
+  # (see Rotation), and the others follow before the next tier, unless a step in the tier sets rotate: false.
+  # order: is the deprecated name of priority:. forward_rejected: false starts a step from the original input
+  # alone, without the previous step's rejected output.
   module ModelPath
     # One attempt: the model, its provider, the fully resolved generation params, and whether the previous
     # step's rejected output is shown to it.
@@ -21,7 +23,7 @@ module Squishling
       end
     end
 
-    STEP_KEYS = %i[model provider params attempts order forward_rejected].freeze
+    STEP_KEYS = %i[model provider params attempts priority order rotate forward_rejected].freeze
 
     module_function
 
@@ -57,43 +59,62 @@ module Squishling
     end
 
     # One Step per attempt. The level's provider applies to steps that don't name their own, and each
-    # step's params are merged over the level-resolved params.
-    def steps(path, provider:, params:)
-      path.flat_map do |step|
+    # step's params are merged over the level-resolved params. turn is this call's place in the method's rotation
+    # (see Rotation): each tier of tied priorities starts turn steps further along.
+    def steps(path, provider:, params:, turn: 0)
+      rotated(path, turn).flat_map do |step|
         attempt = Step.new(model: step[:model], provider: step[:provider] || provider,
           params: Params.resolve(params, step[:params]), forward_rejected: step[:forward_rejected])
         [attempt] * step[:attempts]
       end
     end
 
+    # Whether any tier of the path has several models to rotate between.
+    def rotating?(path)
+      tiers(path).any? { |tier| rotates?(tier) }
+    end
+
+    # Steps sorted by priority, lowest first, ties kept in the order declared.
     def ordered(steps, label)
-      numbered = steps.count { |step| step.key?(:order) }
-      return steps.map { |step| step.except(:order) } if numbered.zero?
+      numbered = steps.count { |step| step.key?(:priority) }
+      return steps if numbered.zero?
 
       if numbered < steps.size
-        raise ConfigurationError, "#{label}: give every step an order: or none (#{numbered} of #{steps.size} have one)"
+        raise ConfigurationError,
+          "#{label}: give every step a priority: or none (#{numbered} of #{steps.size} have one)"
       end
 
-      orders = steps.map { |step| step[:order] }
-      duplicates = orders.tally.select { |_, count| count > 1 }.keys
-      raise ConfigurationError, "#{label}: duplicate order: #{duplicates.join(', ')}" if duplicates.any?
+      steps.each_with_index.sort_by { |step, index| [step[:priority], index] }.map(&:first)
+    end
+
+    # Runs of steps with the same priority. A step without one is a tier of its own.
+    def tiers(path)
+      path.slice_when { |before, after| before[:priority].nil? || before[:priority] != after[:priority] }
+    end
+
+    def rotates?(tier)
+      tier.size > 1 && tier.none? { |step| step[:rotate] == false }
+    end
 
-      steps.sort_by { |step| step[:order] }.map { |step| step.except(:order) }
+    def rotated(path, turn)
+      tiers(path).flat_map { |tier| rotates?(tier) ? tier.rotate(turn % tier.size) : tier }
     end
 
-    def normalize_step(step, label)
+    # extras: names more keys a step Hash may carry for its caller (a judge's type:, say). They are kept as given,
+    # for the caller to check.
+    def normalize_step(step, label, extras: [])
       case step
       when String, Symbol
         { model: model_name(step, label), provider: nil, params: nil, attempts: 1, forward_rejected: true }
-      when Hash then normalize_hash(step, label)
+      when Hash then normalize_hash(step, label, extras)
       else
         raise ConfigurationError, "#{label}: each step must be a model name or a Hash with model:, got #{step.inspect}"
       end
     end
 
-    def normalize_hash(step, label)
+    def normalize_hash(step, label, extras)
       step = step.transform_keys(&:to_sym)
-      unknown = step.keys - STEP_KEYS
+      unknown = step.keys - STEP_KEYS - extras
       raise ConfigurationError, "#{label}: unknown step option(s) #{unknown.join(', ')}" if unknown.any?
       unless step[:model].is_a?(String) || step[:model].is_a?(Symbol)
         raise ConfigurationError, "#{label}: a step needs a model: name, got #{step.inspect}"
@@ -104,8 +125,13 @@ module Squishling
       unless attempts.is_a?(Integer) && attempts.positive?
         raise ConfigurationError, "#{label} (#{model}): attempts: must be a positive Integer, got #{attempts.inspect}"
       end
-      if step.key?(:order) && !step[:order].is_a?(Integer)
-        raise ConfigurationError, "#{label} (#{model}): order: must be an Integer, got #{step[:order].inspect}"
+
+      step = priority_alias(step, label, model)
+      if step.key?(:priority) && !step[:priority].is_a?(Integer)
+        raise ConfigurationError, "#{label} (#{model}): priority: must be an Integer, got #{step[:priority].inspect}"
+      end
+      if step.key?(:rotate) && ![true, false].include?(step[:rotate])
+        raise ConfigurationError, "#{label} (#{model}): rotate: must be true or false, got #{step[:rotate].inspect}"
       end
 
       forward_rejected = step.fetch(:forward_rejected, true)
@@ -116,7 +142,24 @@ module Squishling
 
       params = step[:params] && Params.normalize(step[:params], "#{label} (#{model}) params")
       normalized = { model:, provider: step[:provider], params:, attempts:, forward_rejected: }
-      step.key?(:order) ? normalized.merge(order: step[:order]) : normalized
+      normalized.merge(step.slice(:priority, :rotate, *extras))
+    end
+
+    # A step used on its own (a judge, a harness's step) can't take part in a tier, so it takes no priority: or
+    # rotate:. who names it for the message.
+    def single_step!(step, who)
+      return unless step.key?(:priority) || step.key?(:rotate)
+
+      raise ConfigurationError, "#{who} takes no priority: or rotate:"
+    end
+
+    # order: is the deprecated name of priority:.
+    def priority_alias(step, label, model)
+      return step unless step.key?(:order)
+      raise ConfigurationError, "#{label} (#{model}): give priority: or order:, not both" if step.key?(:priority)
+
+      warn "[Squishling] #{label} (#{model}): order: is deprecated, use priority:"
+      step.except(:order).merge(priority: step[:order])
     end
 
     def model_name(model, label)