squishling 0.4.0 → 0.5.0

data/lib/squishling/output_check.rb

@@ -5,6 +5,7 @@ module Squishling
   # method's squish_validate check.
   class OutputCheck
     MAX_ERRORS = 20
+    EMPTY = "response was empty"
     UNREPRESENTABLE = "response contained a value JSON can't represent (such as Infinity or invalid UTF-8)"
 
     def initialize(definition, receiver, inputs)
@@ -13,32 +14,34 @@ module Squishling
       @inputs = inputs
     end
 
-    # Parses and validates a response: [typed result, []] when it passes, [nil or result, errors] otherwise.
+    # Parses and validates a response: [typed result, []] when it passes, [nil or result, error Details] otherwise.
     # The judge's role skips squish_validate, which checks the operation's output, not a verdict.
     def call(content, prompt)
       data, errors = parse(content)
-      errors = prompt.schema.validate(data) if errors.empty?
+      errors = InvalidOutputError::Detail.from(prompt.schema.validate(data), :schema) if errors.empty?
       return [nil, cap(errors)] if errors.any?
 
       result = prompt.schema.build(data, squished: true)
-      [result, prompt.role == :judge ? [] : validator_errors(result)]
+      [result, prompt.role == :judge ? [] : InvalidOutputError::Detail.from(validator_errors(result), :validator)]
     end
 
     private
 
-    # RubyLLM parses structured output itself and leaves the raw string when that fails, so
+    # An adapter may parse structured output itself (RubyLLM does) and leave the raw string when that fails, so
     # content is a Hash on success, or a String/nil when the model refused, was cut off, or
     # ignored the schema.
     def parse(content)
-      return [nil, [UNREPRESENTABLE]] if content.is_a?(String) && !content.valid_encoding?
-      return [nil, ["response was empty"]] if content.nil? || (content.is_a?(String) && content.strip.empty?)
+      return [nil, [parse_error(UNREPRESENTABLE)]] if content.is_a?(String) && !content.valid_encoding?
+      return [nil, [parse_error(EMPTY)]] if content.nil? || (content.is_a?(String) && content.strip.empty?)
 
       data = content.is_a?(String) ? JSON.parse(strip_code_fence(content)) : content
-      representable?(data) ? [data, []] : [nil, [UNREPRESENTABLE]]
+      representable?(data) ? [data, []] : [nil, [parse_error(UNREPRESENTABLE)]]
     rescue JSON::ParserError => e
-      [nil, ["response was not valid JSON#{parse_position(e)}"]]
+      [nil, [parse_error("response was not valid JSON#{parse_position(e)}")]]
     end
 
+    def parse_error(message) = InvalidOutputError::Detail.new(message, :parse)
+
     # JSON.parse turns an out-of-range number such as 1e400 into Infinity, and accepts invalid UTF-8 in a string.
     # Both can satisfy the schema but can't be serialized again (and the deterministic path rejects them), so
     # they are invalid output.
@@ -54,7 +57,8 @@ module Squishling
     def cap(errors)
       return errors if errors.size <= MAX_ERRORS
 
-      errors.first(MAX_ERRORS) + ["... and #{errors.size - MAX_ERRORS} more errors"]
+      notice = "... and #{errors.size - MAX_ERRORS} more errors"
+      errors.first(MAX_ERRORS) + [InvalidOutputError::Detail.new(notice, :squishling)]
     end
 
     # The parser's message quotes a snippet of the response, which must not reach error messages or logs

data/lib/squishling/params.rb

@@ -5,20 +5,19 @@ module Squishling
   module Params
     THINKING_KEYS = %i[effort budget display].freeze
 
-    # Request keys Squishling or RubyLLM own. Provider options are merged into the request last,
-    # overriding RubyLLM's defaults, so these would silently replace the model, the conversation,
-    # or the strict output format.
+    # Request keys Squishling or the adapter own. Provider params are merged into the request alongside what the
+    # adapter builds, so these would silently replace the model, the conversation, or the strict output format.
     RESERVED_KEYS = %i[
       model messages input inputs instructions contents system system_instruction systemInstruction cachedContent
       stream stream_options store include response_format text output_config outputConfig tools tool_choice
-      toolConfig tool_config schema
+      toolConfig tool_config schema functions function_call
     ].freeze
 
     # Some protocols carry the structured-output format (or tool selection) inside a container that is
     # itself allowed, because it also holds ordinary settings (Gemini's topK, Mistral's top_p, ...). RubyLLM
     # deep-merges provider options, so these nested keys would override the strict format just like the
     # top-level ones. Re-check each protocol's render_payload (ruby_llm protocols/*/chat.rb) on every
-    # ruby_llm upgrade.
+    # ruby_llm upgrade (the RubyLLM adapter is where they matter).
     GENERATION_CONFIG_RESERVED_KEYS = %i[
       responseMimeType response_mime_type responseSchema response_schema responseJsonSchema response_json_schema
       response_format tools tool_choice toolConfig tool_config
@@ -49,11 +48,11 @@ module Squishling
       return if reserved.empty?
 
       raise ConfigurationError, "#{label}: #{reserved.join(', ')} can't be set through params " \
-                                "(controlled by Squishling/RubyLLM#{hint})"
+                                "(controlled by Squishling and the adapter#{hint})"
     end
 
     # Nested keys are compared as symbols so a string-keyed "responseMimeType" can't slip past the check.
-    # A non-Hash container would replace (not merge into) the one RubyLLM builds, wiping the strict format.
+    # A non-Hash container would replace (not merge into) the one the adapter builds, wiping the strict format.
     # Validated containers are copied and frozen so they can't be mutated after the check.
     def reject_nested_reserved!(params, label)
       nested = NESTED_RESERVED_KEYS.flat_map do |container, reserved|
@@ -76,22 +75,6 @@ module Squishling
       layers.compact.reduce({}) { |merged, layer| merged.merge(layer) }.compact
     end
 
-    # Portable settings use RubyLLM's dedicated setters, which translate them for each provider;
-    # everything else is merged into the provider request as-is.
-    def apply(chat, params)
-      rest = params.except(:temperature, :thinking, :max_output_tokens)
-      chat.with_temperature(params[:temperature]) if params.key?(:temperature)
-      chat.with_max_output_tokens(params[:max_output_tokens]) if params.key?(:max_output_tokens)
-      apply_thinking(chat, params[:thinking]) if params.key?(:thinking)
-      chat.with_provider_options(rest) if rest.any?
-      chat
-    end
-
-    # true: the model's default thinking; false: off; a Hash: { effort:, budget:, display: }.
-    def apply_thinking(chat, thinking)
-      thinking.is_a?(Hash) ? chat.with_thinking(**thinking) : chat.with_thinking(thinking)
-    end
-
     def normalize_thinking(thinking, label)
       return thinking if [true, false].include?(thinking)
 

data/lib/squishling/railtie.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Squishling
+  # Adds the squishling:verify rake task to a Rails app (loaded only when Rails is).
+  class Railtie < ::Rails::Railtie
+    rake_tasks do
+      load File.expand_path("tasks/squishling.rake", __dir__)
+    end
+  end
+end

data/lib/squishling/result.rb

@@ -4,6 +4,17 @@ module Squishling
   # Builds Data classes from a JSON Schema so both the LLM and deterministic paths return
   # the same typed object.
   module Result
+    # Property names a generated class can't take: a Data member of the same name replaces the method, which the
+    # result (or Squishling) needs. result_spec.rb probes every Object, Kernel, and Data method, so a method a newer
+    # Ruby adds can't slip through unreserved.
+    RESERVED_NAMES = %i[
+      squished? squishling_squished []
+      class is_a? kind_of? inspect to_s to_h hash == eql? members deconstruct_keys with dup
+      send public_send
+      __send__ object_id __id__
+      initialize initialize_copy initialize_dup initialize_clone respond_to_missing?
+    ].freeze
+
     # Mixed into every generated result class.
     module Instance
       def initialize(squishling_squished: false, **attrs)
@@ -28,15 +39,9 @@ module Squishling
     module ClassMethods
       def from_h(hash, squished: false)
         hash = hash.transform_keys(&:to_sym)
-        attrs = members.to_h { |member| [member, squishling_fields[member].call(hash[member], squished)] }
+        attrs = members.to_h { |member| [member, @squishling_fields[member].call(hash[member], squished)] }
         new(squishling_squished: squished, **attrs)
       end
-
-      private
-
-      def squishling_fields
-        @squishling_fields ||= @squishling_builder.fields_for(@squishling_schema)
-      end
     end
 
     class << self
@@ -67,21 +72,37 @@ module Squishling
         schema = resolve(schema)
         return nil unless schema.is_a?(Hash) && schema["properties"].is_a?(Hash)
 
-        @classes[schema] ||= begin
-          klass = Data.define(*schema["properties"].keys.map(&:to_sym))
-          klass.include(Instance)
-          klass.extend(ClassMethods)
-          klass.instance_variable_set(:@squishling_builder, self)
-          klass.instance_variable_set(:@squishling_schema, schema)
-          klass
-        end
+        return @classes[schema] if @classes.key?(schema)
+
+        klass = @classes[schema] = define_class(schema)
+        # Memoized first so recursive schemas stop; typing the fields builds nested classes now, so a bad
+        # property name in any object the result types (nested, array items, resolved $refs) raises at schema build.
+        klass.instance_variable_set(:@squishling_fields, fields_for(schema))
+        klass
       end
 
+      private
+
       def fields_for(schema)
         schema["properties"].to_h { |key, property| [key.to_sym, converter_for(property)] }
       end
 
-      private
+      def define_class(schema)
+        klass = Data.define(*property_names(schema))
+        klass.include(Instance)
+        klass.extend(ClassMethods)
+        klass
+      end
+
+      def property_names(schema)
+        names = schema["properties"].keys.map(&:to_sym)
+        clashes = names & RESERVED_NAMES
+        return names if clashes.empty?
+
+        raise ConfigurationError,
+          "output_schema property names clash with methods the result needs: #{clashes.join(', ')}. " \
+          "Rename the field(s) (e.g. `class` to `ticket_class`)"
+      end
 
       def converter_for(property)
         property = resolve(property)

data/lib/squishling/rotation.rb

@@ -12,6 +12,11 @@ module Squishling
     private_constant :COUNTS, :COUNTS_LOCK
 
     class << self
+      # The turn the next call will get, without counting it.
+      def peek(klass, name)
+        COUNTS_LOCK.synchronize { (COUNTS[klass] || {}).fetch(name, 0) }
+      end
+
       # This call's turn, counting from 0, and counts it.
       def advance(klass, name)
         COUNTS_LOCK.synchronize do

data/lib/squishling/rspec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "rspec/expectations"
+require "squishling"
+
+# be_squishling_valid: passes when Squishling.verify finds no problems, and fails with its report.
+#   expect(Squishling).to be_squishling_valid   # every squished class (eager load first in Rails)
+#   expect(InvoiceParser).to be_squishling_valid
+#   expect([InvoiceParser, Triage]).to be_squishling_valid
+RSpec::Matchers.define :be_squishling_valid do
+  match do |actual|
+    @report = actual.equal?(Squishling) ? Squishling.verify : Squishling.verify(actual)
+    @report.ok?
+  end
+
+  failure_message { @report.to_s }
+  failure_message_when_negated { "expected Squishling.verify to find problems, but it found none" }
+end

data/lib/squishling/run/messages.rb

@@ -4,10 +4,48 @@ module Squishling
   class Run
     # The follow-up messages a Run sends a model after an output was rejected.
     module Messages
+      RETRY_HEAD = "Your previous response was rejected:\n- "
+      RETRY_TAIL = "\nRespond again with corrected JSON only."
+      RETURNED = "\n\nA previous attempt at this request returned:\n"
+      REJECTED = "\nIt was rejected:\n- "
+      ESCALATION_TAIL = "\nRespond with corrected JSON only."
+      private_constant :RETRY_HEAD, :RETRY_TAIL, :RETURNED, :REJECTED, :ESCALATION_TAIL
+
+      # What an escalation message carried: the original input, the (capped) rejected output text, and the errors.
+      Escalation = Data.define(:input, :rejected, :errors)
+
+      RETRY_PATTERN = /\A#{Regexp.escape(RETRY_HEAD)}(.*)#{Regexp.escape(RETRY_TAIL)}\z/m
+      ESCALATION_PATTERN = Regexp.new(
+        "\\A(.*?)#{Regexp.escape(RETURNED)}(.*)#{Regexp.escape(REJECTED)}(.*)#{Regexp.escape(ESCALATION_TAIL)}\\z",
+        Regexp::MULTILINE
+      )
+      private_constant :RETRY_PATTERN, :ESCALATION_PATTERN
+
+      class << self
+        # The errors a retry message carries, or nil when text isn't one. The inverse of the message a Run sends, kept
+        # here so the two can't drift apart (the test helpers read what was sent through it). Errors are told apart
+        # by splitting on "\n- ", so one that itself contains that sequence reads back as several.
+        def parse_retry(text)
+          text.match(RETRY_PATTERN)&.then { |match| match[1].split("\n- ") }
+        end
+
+        # What an escalation message carried, or nil when text isn't one.
+        def parse_escalation(text)
+          match = text.match(ESCALATION_PATTERN)
+          match && Escalation.new(input: match[1], rejected: match[2], errors: match[3].split("\n- "))
+        end
+      end
+
       private
 
       def retry_message(errors)
-        "Your previous response was rejected:\n- #{errors.join("\n- ")}\nRespond again with corrected JSON only."
+        "#{RETRY_HEAD}#{errors.join("\n- ")}#{RETRY_TAIL}"
+      end
+
+      # The first message to a chat: the input, along with the output rejected before it (if any) unless the step
+      # opts out of seeing it.
+      def fresh_message(step, input, rejected)
+        rejected && step.forward_rejected ? escalation_message(input, *rejected) : input
       end
 
       # The first message to a fresh chat after an earlier model's output was rejected. The rejected output is
@@ -19,8 +57,7 @@ module Squishling
           omitted = previous.length - Invoker::MAX_FORWARDED_CHARS
           previous = "#{previous[0, Invoker::MAX_FORWARDED_CHARS]}... [truncated, #{omitted} more characters]"
         end
-        "#{input}\n\nA previous attempt at this request returned:\n#{previous}\n" \
-          "It was rejected:\n- #{errors.join("\n- ")}\nRespond with corrected JSON only."
+        "#{input}#{RETURNED}#{previous}#{REJECTED}#{errors.join("\n- ")}#{ESCALATION_TAIL}"
       end
 
       # The rejected output as text. Output that was rejected for being unrepresentable in JSON (Infinity, invalid

data/lib/squishling/run/rejections.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Squishling
+  class Run
+    # What a Run does with a reply it rejects: the attempt that follows, the error it raises, the reason it logs, and
+    # the token budget that can end the call before another request. A model that sends back the reply it was just told
+    # was invalid, in the same chat, would do it again, so a repeated reply gives up on the rest of its step's attempts.
+    module Rejections
+      private
+
+      # The number of the attempt after attempt in path, or nil when none is left.
+      def next_attempt(path, attempt, repeated)
+        return (attempt + 1 if attempt < path.size) unless repeated
+
+        skipped = path.drop(attempt).index { |step| step != path[attempt - 1] }
+        attempt + skipped + 1 if skipped
+      end
+
+      def invalid_output(response, errors, attempt, models, source = "LLM")
+        InvalidOutputError.new(errors:, raw: response.content, attempts: attempt, models:, source:,
+          finish_reason: response.finish_reason)
+      end
+
+      # Why an attempt was rejected, for the log: the output check's errors, plus a note when the reply was cut off.
+      def rejection_reason(errors, response)
+        reason = errors.join("; ")
+        response.truncated? ? "#{reason} (#{InvalidOutputError::CUTOFF_NOTE})" : reason
+      end
+
+      # Raises BudgetExhaustedError when the call has spent its token budget (see Budget), before another request.
+      # attempts is how many were made so far, models the ones tried, rejected the last [raw content, errors].
+      def check_budget!(attempts: 0, models: nil, rejected: nil)
+        exhausted = @client.exhausted_budget
+        return unless exhausted
+
+        error = BudgetExhaustedError.new(**exhausted, errors: rejected&.last || [], raw: rejected&.first,
+          attempts: (attempts if attempts.positive?), models: (models.dup if models&.any?))
+        log("stopped, #{error.errors.first}")
+        raise error
+      end
+    end
+  end
+end

data/lib/squishling/run.rb

@@ -13,11 +13,12 @@ module Squishling
   # on other threads; parsing, validation, and every callback (squish_validate, compare:, procs) stay on this one.
   class Run
     include Messages
+    include Rejections
 
-    private_constant :Messages
+    private_constant :Messages, :Rejections
 
     # One of #samples' outputs, between rounds.
-    Sample = Struct.new(:name, :stage, :chat, :rejected, :result)
+    Sample = Struct.new(:name, :stage, :chat, :rejected, :result, :reask)
 
     SAMPLE_NAMES = ("a".."z").to_a.freeze
     private_constant :Sample, :SAMPLE_NAMES
@@ -32,8 +33,9 @@ module Squishling
       @payload = payload
       @purpose = purpose
       @harness = harness
-      @steps = definition.escalation_path.freeze
-      @client = LLMClient.new(definition.label, purpose:, harness: harness.type, handed_off: definition.handed_off)
+      @steps = definition.escalation_path(receiver, inputs).freeze
+      @client = LLMClient.new(definition.label, purpose:, harness: harness.type, handed_off: definition.handed_off,
+        budget: definition.budget)
       @output_check = OutputCheck.new(definition, receiver, inputs)
       @owner = Thread.current
       @produced = {}.compare_by_identity
@@ -54,8 +56,9 @@ module Squishling
       @stages ||= steps.slice_when { |before, after| before != after }.map(&:freeze).freeze
     end
 
-    # A stage from a model name or a step Hash (model:, provider:, params:, attempts:, forward_rejected:), the
-    # way a judge step is resolved: it keeps its own provider, and its params merge over the method's.
+    # A stage from a model name or a step Hash (model:, provider:, params:, attempts:, forward_rejected:,
+    # retry_temperature:), the way a judge step is resolved: it keeps its own provider, and its params merge over
+    # the method's.
     # A stage (an Array of steps) is returned as is, and a single step is a stage of one attempt.
     def stage(declaration)
       return checked_stage([declaration]) if declaration.is_a?(ModelPath::Step)
@@ -71,9 +74,11 @@ module Squishling
     end
 
     # Makes each attempt of the stages in order (the escalation's steps when none are given) until an output
-    # passes the schema and squish_validate. Consecutive attempts on the same step continue the same chat, so the
+    # passes the schema and squish_validate. Raises BudgetExhaustedError instead of making an attempt once the
+    # call's token budget (see Budget) is spent. Consecutive attempts on the same step continue the same chat, so the
     # model sees what it got wrong; a new step starts a fresh chat told about the last rejected output (unless
-    # the step sets forward_rejected: false). Raises InvalidOutputError, or LLMError, when the last attempt fails.
+    # the step sets forward_rejected: false). A retry that repeats the rejected output it was just told about gives
+    # up on the rest of its step's attempts. Raises InvalidOutputError, or LLMError, when the last attempt fails.
     def escalate(*stages)
       owner!
       path = stages.empty? ? steps : stages.flat_map { |declared| stage(declared) }
@@ -103,6 +108,8 @@ module Squishling
         pending = samples.reject(&:result)
         break if pending.empty?
 
+        check_budget!(attempts: attempt - 1, models: samples.first.stage.first(attempt - 1).map(&:model),
+          rejected: pending.filter_map(&:rejected).last)
         round(pending, attempt)
       end
       samples.map(&:result)
@@ -119,7 +126,7 @@ module Squishling
       judge = judge_step(judge) unless judge.nil?
       Judge.new(definition: @definition, receiver: @receiver, client: @client, judge:, instructions:, path: steps,
         purpose: @purpose, payload: @payload, escalate: method(:escalate_with), log_failure: method(:log_failure),
-        guard: method(:owner!), harness_type: @harness.type)
+        guard: method(:owner!), budget_check: method(:check_budget!), harness_type: @harness.type)
     end
 
     # Whether two outputs agree: compare (a Proc, evaluated against the receiver with the two results and the
@@ -167,42 +174,49 @@ module Squishling
       chat = nil
       rejected = nil # [raw content, errors] of the last invalid output
       models = []
+      attempt = 1
 
-      path.each.with_index(1) do |step, attempt|
+      while attempt <= path.size
+        check_budget!(attempts: models.size, models:, rejected:)
+        step = path[attempt - 1]
         models << step.model
         last = attempt == path.size
-        if chat && step == path[attempt - 2]
+        reask = !chat.nil? && step == path[attempt - 2]
+        if reask
           message = retry_message(rejected.last)
         else
           chat = start_chat(step, prompt)
-          message = rejected && step.forward_rejected ? escalation_message(prompt.input, *rejected) : prompt.input
+          message = fresh_message(step, prompt.input, rejected)
         end
 
         begin
-          response = @client.ask(chat, message, step, attempt:, attempts: path.size, role: prompt.role)
+          response = @client.ask(chat, message, step, attempt:, attempts: path.size, role: prompt.role, reask:)
         rescue LLMError => e
-          squawk(prompt, path, attempt, nil, e)
+          squawk(prompt, path, attempt, nil, e, reask:)
           raise if last
 
-          # The failed request may be half-recorded in this chat, so the next attempt starts a fresh one.
+          # A failed request starts the next attempt in a fresh chat, as a new step does.
           chat = nil
           log_failure(prompt.role, attempt, path, e.message)
+          attempt += 1
           next
         end
 
         result, errors = @output_check.call(response.content, prompt)
         if errors.empty?
-          squawk(prompt, path, attempt, response, nil)
+          squawk(prompt, path, attempt, response, nil, reask:)
           return produce(result, prompt)
         end
 
-        error = InvalidOutputError.new(errors:, raw: response.content, attempts: attempt, models: models.dup,
-          source: prompt.role == :judge ? "Judge" : "LLM")
-        squawk(prompt, path, attempt, response, error)
-        raise error if last
+        repeated = reask && response.content == rejected.first
+        following = next_attempt(path, attempt, repeated)
+        error = invalid_output(response, errors, models.size, models.dup, prompt.role == :judge ? "Judge" : "LLM")
+        squawk(prompt, path, attempt, response, error, reask:, repeated:, final: following.nil?)
+        raise error unless following
 
-        rejected = [response.content, errors]
-        log_failure(prompt.role, attempt, path, errors.join("; "))
+        rejected = [response.content, error.errors]
+        log_failure(prompt.role, attempt, path, rejection_reason(error.errors, response), following:, repeated:)
+        attempt = following
       end
     end
 
@@ -213,9 +227,10 @@ module Squishling
         message = sample_message(sample, step)
         chat = sample.chat
         attempts = sample.stage.size
-        -> { @client.ask(chat, message, step, attempt:, attempts:, role: :sample, sample: sample.name) }
+        reask = sample.reask
+        -> { @client.ask(chat, message, step, attempt:, attempts:, role: :sample, sample: sample.name, reask:) }
       end
-      outcomes = LLMClient.concurrently(jobs)
+      outcomes = @client.concurrently(jobs)
       # A setup mistake (or anything that isn't a failed request) ends the call, whichever sample hit it.
       fatal = outcomes.map(&:last).find { |error| error && !error.is_a?(LLMError) }
       raise fatal if fatal
@@ -225,10 +240,11 @@ module Squishling
 
     # The next message for a sample: a retry in its chat after invalid output, otherwise a fresh chat.
     def sample_message(sample, step)
-      return retry_message(sample.rejected.last) if sample.chat
+      sample.reask = !sample.chat.nil?
+      return retry_message(sample.rejected.last) if sample.reask
 
       sample.chat = start_chat(step, @prompt)
-      sample.rejected && step.forward_rejected ? escalation_message(@prompt.input, *sample.rejected) : @prompt.input
+      fresh_message(step, @prompt.input, sample.rejected)
     end
 
     def record(sample, (response, error), attempt)
@@ -236,7 +252,7 @@ module Squishling
       last = attempt == stage.size
       role = "sample #{sample.name}"
       if error
-        squawk(@prompt, stage, attempt, nil, error, sample:)
+        squawk(@prompt, stage, attempt, nil, error, sample:, reask: sample.reask)
         raise error if last
 
         sample.chat = nil
@@ -245,17 +261,19 @@ module Squishling
 
       result, errors = @output_check.call(response.content, @prompt)
       if errors.empty?
-        squawk(@prompt, stage, attempt, response, nil, sample:)
+        squawk(@prompt, stage, attempt, response, nil, sample:, reask: sample.reask)
         return sample.result = produce(result, @prompt)
       end
 
-      invalid = InvalidOutputError.new(errors:, raw: response.content, attempts: attempt,
-        models: stage.first(attempt).map(&:model))
-      squawk(@prompt, stage, attempt, response, invalid, sample:)
-      raise invalid if last
+      # A sample is one step (#samples refuses a stage that mixes them), so a repeat leaves nothing to move on to.
+      repeated = sample.reask && response.content == sample.rejected.first
+      invalid = invalid_output(response, errors, attempt, stage.first(attempt).map(&:model))
+      squawk(@prompt, stage, attempt, response, invalid, sample:, reask: sample.reask, repeated:,
+        final: last || repeated)
+      raise invalid if last || repeated
 
-      sample.rejected = [response.content, errors]
-      log_failure(role, attempt, stage, errors.join("; "))
+      sample.rejected = [response.content, invalid.errors]
+      log_failure(role, attempt, stage, rejection_reason(invalid.errors, response))
     end
 
     # Remembers a result of the method's own output (a judge's verdict isn't one), so a harness can only return
@@ -271,25 +289,31 @@ module Squishling
 
     # Runs the observability hook, if any, with this attempt's raw output (nil when the call itself failed),
     # the error that ended it (nil when it was accepted), and what was asked. role is nil for escalation
-    # attempts, :sample (with the sample's name) or :judge.
-    def squawk(prompt, path, attempt, response, error, sample: nil)
+    # attempts, :sample (with the sample's name) or :judge. reask is whether the attempt retried in the same chat,
+    # repeated whether it sent back the reply the attempt before it was rejected for, and final whether no attempt
+    # follows it.
+    def squawk(prompt, path, attempt, response, error, sample: nil, reask: false, repeated: false,
+      final: attempt == path.size)
       hook = @definition.squawk
-      return unless hook
+      return unless hook && Squawk.report?(@definition.squawk_on, attempt, error, final)
 
       step = path[attempt - 1]
       metadata = {
-        label:, attempt:, attempts: path.size, final: attempt == path.size,
-        model: response&.model || step.model, provider: step.provider, params: step.params, input: prompt.input,
-        usage: response&.tokens&.to_h, role: sample ? :sample : prompt.role, sample: sample&.name
+        label:, attempt:, attempts: path.size, final:, repeated:,
+        model: response&.model || step.model, provider: step.provider, params: step.request_params(reask),
+        input: prompt.input, usage: response&.usage, finish_reason: response&.finish_reason,
+        role: sample ? :sample : prompt.role, sample: sample&.name
       }
       Squawk.call(hook, output: response&.content, metadata:, error:)
     end
 
-    def log_failure(role, attempt, path, reason)
+    # following is the attempt that comes next, which is not the next one when a repeated reply skips ahead.
+    def log_failure(role, attempt, path, reason, following: attempt + 1, repeated: false)
       step = path[attempt - 1]
-      next_step = path[attempt]
+      next_step = path[following - 1]
       action = next_step == step ? "retrying" : "escalating to #{next_step.display_name}"
       who = role ? "#{role} attempt" : "attempt"
+      reason = "it repeated the rejected reply: #{reason}" if repeated
       log("#{who} #{attempt} of #{path.size} (#{step.display_name}) failed, #{action}: #{reason}")
     end
   end

data/lib/squishling/schema.rb

@@ -26,17 +26,19 @@ module Squishling
       end
     end
 
-    # The payload handed to RubyLLM's `with_schema`: always strict, without LOCAL_ONLY_KEYWORDS.
+    # The payload handed to the adapter (Adapter::Request#schema): always strict, without LOCAL_ONLY_KEYWORDS.
     attr_reader :llm_schema
     # The full JSON Schema used for validation and result typing.
     attr_reader :json_schema
+    # The Data class results are built as, or nil when the schema isn't an object with properties.
+    attr_reader :result_class
 
     def initialize(raw)
       hash = self.class.jsonify(to_hash(raw))
       wrapped = wrapped?(hash)
       body = wrapped ? hash["schema"] : hash
 
-      # Same precedence RubyLLM uses: a top-level strict flag wins over one inside the schema.
+      # Same precedence RubyLLM 2.0 uses: a top-level strict flag wins over one inside the schema.
       strict = hash.key?("strict") ? hash["strict"] : body["strict"]
       raise ConfigurationError, "Squishling only supports strict output schemas (got strict: false)" if strict == false
 
@@ -48,22 +50,21 @@ module Squishling
         "strict" => true
       }.compact
       @validator = JSONSchemer.schema(@json_schema)
+      # Built now so a property name the result can't take fails here, not on the first response.
+      @result_class = Result.build(@json_schema)
     end
 
     def validate(data)
-      @validator.validate(data).map { |error| error["error"] }
+      errors = @validator.validate(data).map { |error| error["error"] }
+      # A raw Hash schema can leave out type: "object"; the typed result is built from a Hash, so anything else is
+      # invalid output rather than a NoMethodError out of the builder.
+      errors.empty? && result_class && !data.is_a?(Hash) ? ["root must be an object"] : errors
     end
 
     def build(data, squished:)
       result_class ? result_class.from_h(data, squished:) : data
     end
 
-    def result_class
-      return @result_class if defined?(@result_class)
-
-      @result_class = Result.build(@json_schema)
-    end
-
     private
 
     def to_hash(raw)