squishling 0.4.0 → 0.5.1

data/lib/squishling/run.rb

@@ -13,11 +13,13 @@ module Squishling
   # on other threads; parsing, validation, and every callback (squish_validate, compare:, procs) stay on this one.
   class Run
     include Messages
+    include Rejections
+    include Reports
 
-    private_constant :Messages
+    private_constant :Messages, :Rejections, :Reports
 
     # One of #samples' outputs, between rounds.
-    Sample = Struct.new(:name, :stage, :chat, :rejected, :result)
+    Sample = Struct.new(:name, :stage, :chat, :rejected, :result, :reask)
 
     SAMPLE_NAMES = ("a".."z").to_a.freeze
     private_constant :Sample, :SAMPLE_NAMES
@@ -32,14 +34,17 @@ module Squishling
       @payload = payload
       @purpose = purpose
       @harness = harness
-      @steps = definition.escalation_path.freeze
-      @client = LLMClient.new(definition.label, purpose:, harness: harness.type, handed_off: definition.handed_off)
+      @steps = definition.escalation_path(receiver, inputs).freeze
+      @client = LLMClient.new(definition.label, purpose:, harness: harness.type, handed_off: definition.handed_off,
+        budget: definition.budget)
       @output_check = OutputCheck.new(definition, receiver, inputs)
       @owner = Thread.current
       @produced = {}.compare_by_identity
     end
 
-    # Whether value is a result one of this run's primitives returned (not one built by hand).
+    # Whether value is a result one of this run's primitives returned (not one built by hand). By identity, so it
+    # tells results that are objects (the typed Data of an object schema) apart; a schema that isn't an object
+    # returns the bare JSON value, and an immediate such as 1 or true is the same object in every run.
     def produced?(value)
       @produced.key?(value)
     end
@@ -54,8 +59,9 @@ module Squishling
       @stages ||= steps.slice_when { |before, after| before != after }.map(&:freeze).freeze
     end
 
-    # A stage from a model name or a step Hash (model:, provider:, params:, attempts:, forward_rejected:), the
-    # way a judge step is resolved: it keeps its own provider, and its params merge over the method's.
+    # A stage from a model name or a step Hash (model:, provider:, params:, attempts:, forward_rejected:,
+    # retry_temperature:), the way a judge step is resolved: it keeps its own provider, and its params merge over
+    # the method's.
     # A stage (an Array of steps) is returned as is, and a single step is a stage of one attempt.
     def stage(declaration)
       return checked_stage([declaration]) if declaration.is_a?(ModelPath::Step)
@@ -71,9 +77,11 @@ module Squishling
     end
 
     # Makes each attempt of the stages in order (the escalation's steps when none are given) until an output
-    # passes the schema and squish_validate. Consecutive attempts on the same step continue the same chat, so the
+    # passes the schema and squish_validate. Raises BudgetExhaustedError instead of making an attempt once the
+    # call's token budget (see Budget) is spent. Consecutive attempts on the same step continue the same chat, so the
     # model sees what it got wrong; a new step starts a fresh chat told about the last rejected output (unless
-    # the step sets forward_rejected: false). Raises InvalidOutputError, or LLMError, when the last attempt fails.
+    # the step sets forward_rejected: false). A retry that repeats the rejected output it was just told about gives
+    # up on the rest of its step's attempts. Raises InvalidOutputError, or LLMError, when the last attempt fails.
     def escalate(*stages)
       owner!
       path = stages.empty? ? steps : stages.flat_map { |declared| stage(declared) }
@@ -103,6 +111,8 @@ module Squishling
         pending = samples.reject(&:result)
         break if pending.empty?
 
+        check_budget!(attempts: attempt - 1, models: samples.first.stage.first(attempt - 1).map(&:model),
+          rejected: pending.filter_map(&:rejected).last)
         round(pending, attempt)
       end
       samples.map(&:result)
@@ -119,7 +129,7 @@ module Squishling
       judge = judge_step(judge) unless judge.nil?
       Judge.new(definition: @definition, receiver: @receiver, client: @client, judge:, instructions:, path: steps,
         purpose: @purpose, payload: @payload, escalate: method(:escalate_with), log_failure: method(:log_failure),
-        guard: method(:owner!), harness_type: @harness.type)
+        guard: method(:owner!), budget_check: method(:check_budget!), harness_type: @harness.type)
     end
 
     # Whether two outputs agree: compare (a Proc, evaluated against the receiver with the two results and the
@@ -167,42 +177,49 @@ module Squishling
       chat = nil
       rejected = nil # [raw content, errors] of the last invalid output
       models = []
+      attempt = 1
 
-      path.each.with_index(1) do |step, attempt|
+      while attempt <= path.size
+        check_budget!(attempts: models.size, models:, rejected:)
+        step = path[attempt - 1]
         models << step.model
         last = attempt == path.size
-        if chat && step == path[attempt - 2]
+        reask = !chat.nil? && step == path[attempt - 2]
+        if reask
           message = retry_message(rejected.last)
         else
           chat = start_chat(step, prompt)
-          message = rejected && step.forward_rejected ? escalation_message(prompt.input, *rejected) : prompt.input
+          message = fresh_message(step, prompt.input, rejected)
         end
 
         begin
-          response = @client.ask(chat, message, step, attempt:, attempts: path.size, role: prompt.role)
+          response = @client.ask(chat, message, step, attempt:, attempts: path.size, role: prompt.role, reask:)
         rescue LLMError => e
-          squawk(prompt, path, attempt, nil, e)
+          squawk(prompt, path, attempt, nil, e, reask:)
           raise if last
 
-          # The failed request may be half-recorded in this chat, so the next attempt starts a fresh one.
+          # A failed request starts the next attempt in a fresh chat, as a new step does.
           chat = nil
           log_failure(prompt.role, attempt, path, e.message)
+          attempt += 1
           next
         end
 
         result, errors = @output_check.call(response.content, prompt)
         if errors.empty?
-          squawk(prompt, path, attempt, response, nil)
+          squawk(prompt, path, attempt, response, nil, reask:)
           return produce(result, prompt)
         end
 
-        error = InvalidOutputError.new(errors:, raw: response.content, attempts: attempt, models: models.dup,
-          source: prompt.role == :judge ? "Judge" : "LLM")
-        squawk(prompt, path, attempt, response, error)
-        raise error if last
+        repeated = reask && response.content == rejected.first
+        following = next_attempt(path, attempt, repeated)
+        error = invalid_output(response, errors, models.size, models.dup, prompt.role == :judge ? "Judge" : "LLM")
+        squawk(prompt, path, attempt, response, error, reask:, repeated:, final: following.nil?)
+        raise error unless following
 
-        rejected = [response.content, errors]
-        log_failure(prompt.role, attempt, path, errors.join("; "))
+        rejected = [response.content, error.errors]
+        log_failure(prompt.role, attempt, path, rejection_reason(error.errors, response), following:, repeated:)
+        attempt = following
       end
     end
 
@@ -213,9 +230,10 @@ module Squishling
         message = sample_message(sample, step)
         chat = sample.chat
         attempts = sample.stage.size
-        -> { @client.ask(chat, message, step, attempt:, attempts:, role: :sample, sample: sample.name) }
+        reask = sample.reask
+        -> { @client.ask(chat, message, step, attempt:, attempts:, role: :sample, sample: sample.name, reask:) }
       end
-      outcomes = LLMClient.concurrently(jobs)
+      outcomes = @client.concurrently(jobs)
       # A setup mistake (or anything that isn't a failed request) ends the call, whichever sample hit it.
       fatal = outcomes.map(&:last).find { |error| error && !error.is_a?(LLMError) }
       raise fatal if fatal
@@ -225,10 +243,11 @@ module Squishling
 
     # The next message for a sample: a retry in its chat after invalid output, otherwise a fresh chat.
     def sample_message(sample, step)
-      return retry_message(sample.rejected.last) if sample.chat
+      sample.reask = !sample.chat.nil?
+      return retry_message(sample.rejected.last) if sample.reask
 
       sample.chat = start_chat(step, @prompt)
-      sample.rejected && step.forward_rejected ? escalation_message(@prompt.input, *sample.rejected) : @prompt.input
+      fresh_message(step, @prompt.input, sample.rejected)
     end
 
     def record(sample, (response, error), attempt)
@@ -236,7 +255,7 @@ module Squishling
       last = attempt == stage.size
       role = "sample #{sample.name}"
       if error
-        squawk(@prompt, stage, attempt, nil, error, sample:)
+        squawk(@prompt, stage, attempt, nil, error, sample:, reask: sample.reask)
         raise error if last
 
         sample.chat = nil
@@ -245,17 +264,19 @@ module Squishling
 
       result, errors = @output_check.call(response.content, @prompt)
       if errors.empty?
-        squawk(@prompt, stage, attempt, response, nil, sample:)
+        squawk(@prompt, stage, attempt, response, nil, sample:, reask: sample.reask)
         return sample.result = produce(result, @prompt)
       end
 
-      invalid = InvalidOutputError.new(errors:, raw: response.content, attempts: attempt,
-        models: stage.first(attempt).map(&:model))
-      squawk(@prompt, stage, attempt, response, invalid, sample:)
-      raise invalid if last
+      # A sample is one step (#samples refuses a stage that mixes them), so a repeat leaves nothing to move on to.
+      repeated = sample.reask && response.content == sample.rejected.first
+      invalid = invalid_output(response, errors, attempt, stage.first(attempt).map(&:model))
+      squawk(@prompt, stage, attempt, response, invalid, sample:, reask: sample.reask, repeated:,
+        final: last || repeated)
+      raise invalid if last || repeated
 
-      sample.rejected = [response.content, errors]
-      log_failure(role, attempt, stage, errors.join("; "))
+      sample.rejected = [response.content, invalid.errors]
+      log_failure(role, attempt, stage, rejection_reason(invalid.errors, response))
     end
 
     # Remembers a result of the method's own output (a judge's verdict isn't one), so a harness can only return
@@ -269,27 +290,13 @@ module Squishling
       @client.chat(step, instructions: prompt.instructions, schema: prompt.schema.llm_schema)
     end
 
-    # Runs the observability hook, if any, with this attempt's raw output (nil when the call itself failed),
-    # the error that ended it (nil when it was accepted), and what was asked. role is nil for escalation
-    # attempts, :sample (with the sample's name) or :judge.
-    def squawk(prompt, path, attempt, response, error, sample: nil)
-      hook = @definition.squawk
-      return unless hook
-
-      step = path[attempt - 1]
-      metadata = {
-        label:, attempt:, attempts: path.size, final: attempt == path.size,
-        model: response&.model || step.model, provider: step.provider, params: step.params, input: prompt.input,
-        usage: response&.tokens&.to_h, role: sample ? :sample : prompt.role, sample: sample&.name
-      }
-      Squawk.call(hook, output: response&.content, metadata:, error:)
-    end
-
-    def log_failure(role, attempt, path, reason)
+    # following is the attempt that comes next, which is not the next one when a repeated reply skips ahead.
+    def log_failure(role, attempt, path, reason, following: attempt + 1, repeated: false)
       step = path[attempt - 1]
-      next_step = path[attempt]
+      next_step = path[following - 1]
       action = next_step == step ? "retrying" : "escalating to #{next_step.display_name}"
       who = role ? "#{role} attempt" : "attempt"
+      reason = "it repeated the rejected reply: #{reason}" if repeated
       log("#{who} #{attempt} of #{path.size} (#{step.display_name}) failed, #{action}: #{reason}")
     end
   end

data/lib/squishling/schema.rb

@@ -4,8 +4,15 @@ module Squishling
   # Normalizes the supported output-schema forms into a JSON Schema, validates data against it,
   # and builds typed results.
   class Schema
-    CACHE = {}.compare_by_identity
+    # Held weakly by the raw schema (a Schematist class, a Hash), so classes replaced by code reloading can be
+    # collected along with their schemas. A Hash compares by content, which yields the same schema.
+    CACHE = ObjectSpace::WeakKeyMap.new
     CACHE_LOCK = Mutex.new
+    # What building a validator raises for a $ref (or $schema) it can't resolve. json_schemer's default resolver
+    # refuses remote URIs, so none is fetched.
+    REF_ERRORS = [
+      JSONSchemer::UnknownRef, JSONSchemer::InvalidRefResolution, JSONSchemer::InvalidRefPointer, URI::InvalidURIError
+    ].freeze
 
     # Conditional keywords (Schematist's `given` and `dependent`) that providers' strict modes don't support.
     # They're validated locally and kept out of the schema sent to the LLM.
@@ -26,17 +33,19 @@ module Squishling
       end
     end
 
-    # The payload handed to RubyLLM's `with_schema`: always strict, without LOCAL_ONLY_KEYWORDS.
+    # The payload handed to the adapter (Adapter::Request#schema): always strict, without LOCAL_ONLY_KEYWORDS.
     attr_reader :llm_schema
     # The full JSON Schema used for validation and result typing.
     attr_reader :json_schema
+    # The Data class results are built as, or nil when the schema isn't an object with properties.
+    attr_reader :result_class
 
     def initialize(raw)
       hash = self.class.jsonify(to_hash(raw))
       wrapped = wrapped?(hash)
       body = wrapped ? hash["schema"] : hash
 
-      # Same precedence RubyLLM uses: a top-level strict flag wins over one inside the schema.
+      # Same precedence RubyLLM 2.0 uses: a top-level strict flag wins over one inside the schema.
       strict = hash.key?("strict") ? hash["strict"] : body["strict"]
       raise ConfigurationError, "Squishling only supports strict output schemas (got strict: false)" if strict == false
 
@@ -47,24 +56,56 @@ module Squishling
         "schema" => provider_schema(@json_schema),
         "strict" => true
       }.compact
-      @validator = JSONSchemer.schema(@json_schema)
+      @validator = validator_for(@json_schema)
+      # Built now so a property name the result can't take fails here, not on the first response.
+      @result_class = Result.build(@json_schema)
     end
 
     def validate(data)
-      @validator.validate(data).map { |error| error["error"] }
+      errors = @validator.validate(data).map { |error| error["error"] }
+      # A raw Hash schema's type can be a union that includes "object" (a bare `properties` is refused at build);
+      # the typed result is built from a Hash, so anything else is invalid output rather than a NoMethodError.
+      errors.empty? && result_class && !data.is_a?(Hash) ? ["root must be an object"] : errors
     end
 
     def build(data, squished:)
       result_class ? result_class.from_h(data, squished:) : data
     end
 
-    def result_class
-      return @result_class if defined?(@result_class)
+    private
 
-      @result_class = Result.build(@json_schema)
+    # The json_schemer validator, with every $ref resolved now, so one that points nowhere (a remote URI, a missing
+    # $defs entry) fails at build and in Squishling.verify, not on the first response.
+    def validator_for(json_schema)
+      validator = JSONSchemer.schema(json_schema)
+      resolve_refs!(validator, {}.compare_by_identity)
+      validator
+    rescue *REF_ERRORS => e
+      raise ConfigurationError, "output_schema has a $ref that can't be resolved (#{e.message}); " \
+                                "only references within the schema are supported"
     end
 
-    private
+    # Walks the parsed schema (json_schemer's Schema#parsed and Keyword#ref_schema) and into each target a $ref
+    # resolves to, so a reference inside a subschema kept under any keyword is checked too. seen stops cycles.
+    def resolve_refs!(schema, seen)
+      return if seen.key?(schema)
+
+      seen[schema] = true
+      schema.parsed.each_value do |keyword|
+        resolve_refs!(keyword.ref_schema, seen) if keyword.respond_to?(:ref_schema)
+        subschemas(keyword.parsed).each { |subschema| resolve_refs!(subschema, seen) }
+      end
+    end
+
+    # The Schema objects directly under a parsed keyword: itself, in a list, or in a map of names.
+    def subschemas(parsed)
+      case parsed
+      when JSONSchemer::Schema then [parsed]
+      when Array then parsed.grep(JSONSchemer::Schema)
+      when Hash then parsed.values.grep(JSONSchemer::Schema)
+      else []
+      end
+    end
 
     def to_hash(raw)
       case raw

data/lib/squishling/secret.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Squishling
+  # Holds a credential (an API key, or the headers that may carry one) so a serializer that walks an object's instance
+  # variables can't write it out: to_s, inspect, to_yaml, as_json (so ActiveSupport's instance_values walk too) and
+  # to_json all show "[REDACTED]", and Marshal refuses. The value sits in a closure rather than an instance variable;
+  # #reveal is the only way to read it, and it is for the code that sends the request. Code running in the process can
+  # still call #reveal; this guards against accidental serialization (a logged config, an error tracker, a cache
+  # dump), not against a hostile caller.
+  class Secret
+    REDACTED = "[REDACTED]"
+
+    def initialize(value)
+      @reveal = -> { value }
+      freeze
+    end
+
+    def reveal
+      @reveal.call
+    end
+
+    def inspect
+      REDACTED
+    end
+
+    alias_method :to_s, :inspect
+
+    def encode_with(coder)
+      coder.scalar = REDACTED
+    end
+
+    def as_json(*)
+      REDACTED
+    end
+
+    def to_json(*)
+      REDACTED.to_json
+    end
+
+    def marshal_dump
+      raise TypeError, "Squishling::Secret can't be dumped: it holds a credential"
+    end
+  end
+  private_constant :Secret
+end

data/lib/squishling/squawk.rb

@@ -1,24 +1,57 @@
 # frozen_string_literal: true
 
 module Squishling
-  # The opt-in observability hook: a callable run after every LLM attempt with the attempt's raw output, so
-  # it can be sent somewhere (an error tracker, a tracing tool). It's the one sanctioned way for model output
-  # to leave the process; nothing runs unless one is configured.
+  # The opt-in observability hook: a callable run after every LLM attempt (or only the rejected and retried
+  # ones, see squawk_on:) with the attempt's raw output, so it can be sent somewhere (an error tracker, a
+  # tracing tool). It's the one sanctioned way for model output to leave the process; nothing runs unless one
+  # is configured. A call that a token budget or a disagreement stops gets one more report, with the error and no
+  # output (metadata's stopped:), since neither has an attempt of its own.
   module Squawk
     KEYWORD_TYPES = %i[key keyreq].freeze
     POSITIONAL_TYPES = %i[req opt].freeze
     KEYWORD_CAPTURE_TYPES = %i[key keyreq keyrest].freeze
     KEYWORDS = %i[output metadata error].freeze
+    # The squawk_on: conditions, each answering "does the hook run for this attempt?" from the attempt's 1-based
+    # number (the one its metadata carries), its error (nil when accepted), and whether it is the last one. A stopped
+    # call's report has an error and is final, so it passes every condition whatever its attempt number (0 or more).
+    CONDITIONS = {
+      all: ->(_attempt, _error, _final) { true },
+      rejections: ->(_attempt, error, _final) { !error.nil? },
+      retries: ->(attempt, error, _final) { !error.nil? || attempt > 1 },
+      final: ->(_attempt, _error, final) { final }
+    }.freeze
+    ON = CONDITIONS.keys.freeze
 
     module_function
 
+    # Which attempts reach the hook (squawk_on:): one condition or a list of them, or nil to inherit. :all is
+    # every attempt, :rejections only the ones that errored, :retries those plus any accepted attempt after the
+    # first, and :final the last attempt the escalation allows (metadata's final:), however it ended; an earlier
+    # accepted attempt is not final. The report of a stopped call is final and has an error, so every
+    # condition lets it through. A list runs the hook when any of them holds.
+    def validate_on(value, where)
+      return value if value.nil?
+
+      conditions = value.is_a?(Array) ? value : [value]
+      unless conditions.any? && conditions.all? { |condition| CONDITIONS.key?(condition) }
+        raise ConfigurationError,
+          "#{where} squawk_on must be one or a list of #{ON.map(&:inspect).join(', ')}, got #{value.inspect}"
+      end
+
+      value.is_a?(Array) ? value.dup.freeze : value
+    end
+
+    def report?(on, attempt, error, final)
+      Array(on).any? { |condition| CONDITIONS.fetch(condition).call(attempt, error, final) }
+    end
+
     # The hook itself, or nil/false to inherit/silence. Anything else must respond to call and take keywords
     # only, which is checked here so a mismatch fails at setup, not inside the first attempt.
     def validate(hook, where)
       return hook if hook.nil? || hook == false
 
       unless hook.respond_to?(:call)
-        raise ConfigurationError, "#{where} squawk must respond to call (or be false), got #{hook.inspect}"
+        raise ConfigurationError, "#{where} squawk must respond to call (or be false), got #{Appendices.describe(hook)}"
       end
       if positional_only?(parameters(hook))
         raise ConfigurationError,

data/lib/squishling/tasks/squishling.rake

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Outside Rails, load it from your Rakefile: load "squishling/tasks/squishling.rake"
+unless Rake::Task.task_defined?("squishling:verify")
+  namespace :squishling do
+    desc "Verify every squished class without making a request (see Squishling.verify)"
+    task :verify do
+      # Only loaded classes are known, so load them all first. Rails defines :environment after loading its
+      # railties' tasks, so it can't be a prerequisite declared here.
+      Rake::Task[:environment].invoke if Rake::Task.task_defined?(:environment)
+      Rails.application.eager_load! if defined?(Rails) && Rails.respond_to?(:application) && Rails.application
+
+      report = Squishling.verify
+      puts report
+      abort unless report.ok?
+    end
+  end
+end

data/lib/squishling/testing/adapter.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # A Squishling adapter that answers from a script instead of a provider (see Script). Install it with
+    # Testing.install. It records every request (#requests) and groups them into the chats they belong to
+    # (#conversations), and it is safe to ask from the worker threads a harness's samples run on: everything shared
+    # is taken under one lock, and replies are chosen by (step, sample, attempt), never by arrival order.
+    class Adapter < Squishling::Adapter::Base
+      # One chat: its sample (nil unless a harness sample's) and the turns exchanged so far.
+      Conversation = Struct.new(:index, :sample, :transcript)
+
+      attr_reader :script
+
+      def initialize(script = Script.new)
+        super()
+        @script = script
+        @lock = Mutex.new
+        @requests = []
+        @conversations = []
+      end
+
+      def judgments?
+        true
+      end
+
+      # Every request so far, in the order they arrived (Testing::Request).
+      def requests
+        @lock.synchronize { @requests.dup }
+      end
+
+      # The requests of each chat, in order: a retry on the same step shares a chat, a new step starts one.
+      def conversations
+        requests.select(&:conversation).group_by(&:conversation).sort.map(&:last)
+      end
+
+      def requests_for(role: nil, sample: nil)
+        requests.select do |request|
+          (role.nil? || request.role == role) && (sample.nil? || request.sample == sample.to_s)
+        end
+      end
+
+      # The models the requests went to, in order (nil for a step that names none).
+      def models
+        requests.map(&:model)
+      end
+
+      # The first request's arguments: what left the process.
+      def arguments
+        requests.first&.arguments
+      end
+
+      def complete(request)
+        outcome = @lock.synchronize { answer(request) }
+        raise outcome if outcome.is_a?(Exception)
+
+        outcome
+      end
+
+      def judge(input, questions:, step:, label:)
+        outcome = @lock.synchronize { judged(input, questions, step, label) }
+        raise outcome if outcome.is_a?(Exception)
+
+        outcome
+      end
+
+      private
+
+      def answer(request)
+        call = call_of(request)
+        outcome = outcome_for(:chat, call, request.label, request.model)
+        conversation = conversation_for(request, call[:sample])
+        conversation.transcript.push(*turns(request, outcome)) unless outcome.is_a?(Exception)
+        record(request, call, outcome, conversation.index)
+        outcome
+      end
+
+      def judged(input, _questions, step, label)
+        call = { role: :judge, sample: nil, attempt: Observer.current&.attempt, attempts: Observer.current&.attempts,
+                 model: step.model, provider: step.provider }
+        outcome = outcome_for(:judgment, call, label, step.model)
+        payload = Schema.jsonify(input)
+        @requests << Request.new(label:, type: :judgment, role: :judge, sample: nil, attempt: call[:attempt],
+          attempts: call[:attempts], model: step.model, provider: step.provider, params: step.params,
+          system_prompt: nil, output_schema: nil, messages: [], conversation: nil, payload:, reply: outcome)
+        outcome
+      end
+
+      def outcome_for(kind, call, label, model)
+        rule = @script.rule_for(kind, call)
+        item = rule ? rule.take : Rule::EXHAUSTED
+        return unscripted(kind, call, label, model, rule) if item.equal?(Rule::EXHAUSTED)
+
+        resolve(item, label)
+      end
+
+      def resolve(item, label)
+        case item
+        when Failure then Failures.build(item.intent, label)
+        when Exception, Judgment then item
+        when Class then item <= Exception ? item.new : response(item)
+        when Reply then Squishling::Adapter::Response.new(content: item.content, model: item.model, tokens: item.usage)
+        else response(item)
+        end
+      end
+
+      def response(content)
+        Squishling::Adapter::Response.new(content:)
+      end
+
+      def unscripted(kind, call, label, model, rule)
+        aimed = call.compact.except(:attempts).map { |key, value| "#{key}: #{value.inspect}" }.join(", ")
+        why = if rule
+          "its script #{rule.empty? ? 'has no replies' : 'has no answers left (once)'}"
+        else
+          "no script covers it"
+        end
+        UnscriptedRequestError.new("#{label}: unscripted #{kind} request to #{model.inspect} (#{aimed}): #{why}. " \
+                                   "Scripted: #{@script.describe}")
+      end
+
+      # The role, sample, and attempt of the request: from the observer that wraps it, or, when the adapter was
+      # installed without one, what the request itself shows.
+      def call_of(request)
+        observed = Observer.current
+        base = { model: request.model, provider: request.provider }
+        if observed
+          return base.merge(role: observed.role, sample: observed.sample, attempt: observed.attempt,
+            attempts: observed.attempts)
+        end
+
+        base.merge(role: judge_schema?(request.schema) ? :judge : nil, sample: nil, attempt: nil, attempts: nil)
+      end
+
+      def judge_schema?(schema)
+        schema["schema"] == Schema.for(Judge::VERDICT_SCHEMA).llm_schema["schema"]
+      end
+
+      # A request that continues a chat sends the chat's transcript back, so it finds its conversation by it.
+      def conversation_for(request, sample)
+        history = request.messages[0...-1]
+        found = history.empty? ? nil : @conversations.reverse.find { |c| c.sample == sample && c.transcript == history }
+        found || Conversation.new(@conversations.size, sample, []).tap { |chat| @conversations << chat }
+      end
+
+      def turns(request, response)
+        [{ role: :user, content: request.messages.last.fetch(:content) }, { role: :assistant, content: response.text }]
+      end
+
+      def record(request, call, outcome, conversation)
+        @requests << Request.new(label: request.label, type: :chat, role: call[:role], sample: call[:sample]&.to_s,
+          attempt: call[:attempt], attempts: call[:attempts], model: request.model, provider: request.provider,
+          params: request.params, system_prompt: request.instructions, output_schema: request.schema["schema"],
+          messages: request.messages, conversation:, payload: Request.payload_of(request.messages), reply: outcome)
+      end
+    end
+
+    # The adapter behind Testing.forbid_provider_calls!: any request that reaches it raises ProviderCallError, which
+    # is not a Squishling::Error, so a squish_fallback can't absorb it. It remembers the calls that were attempted,
+    # for a test that rescued the error.
+    class ForbiddenAdapter < Squishling::Adapter::Base
+      def initialize
+        super
+        @lock = Mutex.new
+        @attempts = []
+      end
+
+      # The labels ("Klass#method") of the calls that reached it.
+      def attempts
+        @lock.synchronize { @attempts.dup }
+      end
+
+      def judgments?
+        true
+      end
+
+      def complete(request)
+        refuse(request.label)
+      end
+
+      def judge(_input, label:, **)
+        refuse(label)
+      end
+
+      private
+
+      def refuse(label)
+        @lock.synchronize { @attempts << label }
+        raise ProviderCallError, "#{label} reached the provider, but no LLM call was expected. Script it with " \
+                                 "Squishling::Testing.install, or fix the code under test"
+      end
+    end
+  end
+end