squishling 0.3.0 → 0.5.0

data/lib/squishling/run.rb

@@ -0,0 +1,320 @@
+# frozen_string_literal: true
+
+module Squishling
+  # One squished call, as a harness sees it (see Harness::Base): the primitives that ask the model for outputs,
+  # and everything they do for free: model, provider, and params resolution, schema and squish_validate checks,
+  # retries in the same chat, escalation, squawk reports, and Squishling's own errors when attempts run out.
+  # Each returns typed results, the same type as the deterministic path.
+  #
+  # A stage is the attempts of one escalation step (a model with its attempts:), as an Array of
+  # ModelPath::Step. Primitives take stages, or anything that declares one (a model name or a step Hash).
+  #
+  # Primitives must be called from the thread that runs the harness. Only the provider requests inside them run
+  # on other threads; parsing, validation, and every callback (squish_validate, compare:, procs) stay on this one.
+  class Run
+    include Messages
+    include Rejections
+
+    private_constant :Messages, :Rejections
+
+    # One of #samples' outputs, between rounds.
+    Sample = Struct.new(:name, :stage, :chat, :rejected, :result, :reask)
+
+    SAMPLE_NAMES = ("a".."z").to_a.freeze
+    private_constant :Sample, :SAMPLE_NAMES
+
+    attr_reader :receiver, :inputs, :steps
+
+    def initialize(definition:, receiver:, inputs:, prompt:, payload:, purpose:, harness:)
+      @definition = definition
+      @receiver = receiver
+      @inputs = inputs
+      @prompt = prompt
+      @payload = payload
+      @purpose = purpose
+      @harness = harness
+      @steps = definition.escalation_path(receiver, inputs).freeze
+      @client = LLMClient.new(definition.label, purpose:, harness: harness.type, handed_off: definition.handed_off,
+        budget: definition.budget)
+      @output_check = OutputCheck.new(definition, receiver, inputs)
+      @owner = Thread.current
+      @produced = {}.compare_by_identity
+    end
+
+    # Whether value is a result one of this run's primitives returned (not one built by hand).
+    def produced?(value)
+      @produced.key?(value)
+    end
+
+    # The squished method, as "Class#method".
+    def label
+      @definition.label
+    end
+
+    # The escalation's steps grouped into stages: runs of identical consecutive steps, the attempts of one model.
+    def stages
+      @stages ||= steps.slice_when { |before, after| before != after }.map(&:freeze).freeze
+    end
+
+    # A stage from a model name or a step Hash (model:, provider:, params:, attempts:, forward_rejected:,
+    # retry_temperature:), the way a judge step is resolved: it keeps its own provider, and its params merge over
+    # the method's.
+    # A stage (an Array of steps) is returned as is, and a single step is a stage of one attempt.
+    def stage(declaration)
+      return checked_stage([declaration]) if declaration.is_a?(ModelPath::Step)
+
+      if declaration.is_a?(Array)
+        return checked_stage(declaration) if declaration.any? && declaration.all?(ModelPath::Step)
+
+        raise ConfigurationError, "#{label}: a stage is the steps of one model, got #{declaration.inspect}"
+      end
+
+      step = Harness::OptionTypes.check(:step, declaration, label, :"harness step")
+      ModelPath.steps([step], provider: nil, params: @definition.params).freeze
+    end
+
+    # Makes each attempt of the stages in order (the escalation's steps when none are given) until an output
+    # passes the schema and squish_validate. Raises BudgetExhaustedError instead of making an attempt once the
+    # call's token budget (see Budget) is spent. Consecutive attempts on the same step continue the same chat, so the
+    # model sees what it got wrong; a new step starts a fresh chat told about the last rejected output (unless
+    # the step sets forward_rejected: false). A retry that repeats the rejected output it was just told about gives
+    # up on the rest of its step's attempts. Raises InvalidOutputError, or LLMError, when the last attempt fails.
+    def escalate(*stages)
+      owner!
+      path = stages.empty? ? steps : stages.flat_map { |declared| stage(declared) }
+      escalate_with(path, @prompt)
+    end
+
+    # One sample per stage (the attempts of one model), each asked on its own chat, each retried per its stage's
+    # attempts like #escalate, with the requests of every round sent concurrently. Returns the outputs in order.
+    # Samples are named "a", "b", ... in squawk reports.
+    def samples(stages)
+      owner!
+      raise ConfigurationError, "#{label}: samples takes an Array of stages" unless stages.is_a?(Array)
+
+      stages = stages.map { |declared| stage(declared) }
+      raise ConfigurationError, "#{label}: samples needs at least one stage" if stages.empty?
+      if stages.size > SAMPLE_NAMES.size
+        raise ConfigurationError, "#{label}: samples takes at most #{SAMPLE_NAMES.size} stages"
+      end
+
+      if (mixed = stages.find { |sample_stage| sample_stage.uniq.size > 1 })
+        raise ConfigurationError, "#{label}: a sample is asked on one model, so its stage can't mix steps " \
+                                  "(got #{mixed.map(&:display_name).join(', ')})"
+      end
+
+      samples = stages.zip(SAMPLE_NAMES).map { |sample_stage, name| Sample.new(name, sample_stage) }
+      (1..stages.map(&:size).max).each do |attempt|
+        pending = samples.reject(&:result)
+        break if pending.empty?
+
+        check_budget!(attempts: attempt - 1, models: samples.first.stage.first(attempt - 1).map(&:model),
+          rejected: pending.filter_map(&:rejected).last)
+        round(pending, attempt)
+      end
+      samples.map(&:result)
+    end
+
+    # A judge that picks between two outputs (see Judge#verdict, which returns a Judge::Verdict), or rejects
+    # both. judge: is a model name or a Hash with model:, provider:, params:, and attempts: (as an escalation
+    # step, without priority:), plus type: :chat (default) or :judgment and, for :judgment, min_confidence:.
+    # Without one, the judge is the escalation's next stage after the first. instructions: is a String, or a
+    # Proc evaluated against the receiver, replacing Harness::DEFAULT_JUDGE_INSTRUCTIONS. The judge is resolved
+    # here, so a missing one raises ConfigurationError before any request is made.
+    def judge(judge: nil, instructions: nil)
+      owner!
+      judge = judge_step(judge) unless judge.nil?
+      Judge.new(definition: @definition, receiver: @receiver, client: @client, judge:, instructions:, path: steps,
+        purpose: @purpose, payload: @payload, escalate: method(:escalate_with), log_failure: method(:log_failure),
+        guard: method(:owner!), budget_check: method(:check_budget!), harness_type: @harness.type)
+    end
+
+    # Whether two outputs agree: compare (a Proc, evaluated against the receiver with the two results and the
+    # method's inputs, truthy for agreement) or, without one, whether they are exactly equal.
+    # compare is the developer's own code, so its exceptions propagate unwrapped.
+    def agree?(first, second, compare = nil)
+      owner!
+      return Schema.jsonify(first.to_h) == Schema.jsonify(second.to_h) unless compare
+
+      @receiver.instance_exec(first, second, **@inputs, &compare) ? true : false
+    end
+
+    # Logs a warning, "[Squishling] Class#method message", to Squishling.config.logger, if there is one.
+    def log(message)
+      Squishling.config.logger&.warn("[Squishling] #{label} #{message}")
+    end
+
+    private
+
+    # Steps made or changed by hand get the params checks a declared step does (no reserved keys).
+    def checked_stage(stage)
+      stage.each { |step| Params.normalize(step.params, "#{label} harness step params") }
+      stage.frozen? ? stage : stage.dup.freeze
+    end
+
+    # A judge declaration as the escalation step Judge takes: one step (no priority:), with its type.
+    def judge_step(declaration)
+      step = ModelPath.normalize_step(declaration, "#{label} judge", extras: Judge::STEP_KEYS)
+      ModelPath.single_step!(step, "#{label} judge: a judge is one step, so it")
+
+      type = step.fetch(:type, :chat)
+      type = :chat if type.nil?
+      step.merge(type: type.is_a?(String) ? type.to_sym : type)
+    end
+
+    def owner!
+      return if Thread.current.equal?(@owner)
+
+      raise ConfigurationError, "#{label}: a harness must call its run's primitives from the thread it runs on"
+    end
+
+    # Makes each attempt of the path until an output passes (see #escalate). The judge's prompt (role :judge)
+    # skips squish_validate and names the judge in errors and logs.
+    def escalate_with(path, prompt)
+      chat = nil
+      rejected = nil # [raw content, errors] of the last invalid output
+      models = []
+      attempt = 1
+
+      while attempt <= path.size
+        check_budget!(attempts: models.size, models:, rejected:)
+        step = path[attempt - 1]
+        models << step.model
+        last = attempt == path.size
+        reask = !chat.nil? && step == path[attempt - 2]
+        if reask
+          message = retry_message(rejected.last)
+        else
+          chat = start_chat(step, prompt)
+          message = fresh_message(step, prompt.input, rejected)
+        end
+
+        begin
+          response = @client.ask(chat, message, step, attempt:, attempts: path.size, role: prompt.role, reask:)
+        rescue LLMError => e
+          squawk(prompt, path, attempt, nil, e, reask:)
+          raise if last
+
+          # A failed request starts the next attempt in a fresh chat, as a new step does.
+          chat = nil
+          log_failure(prompt.role, attempt, path, e.message)
+          attempt += 1
+          next
+        end
+
+        result, errors = @output_check.call(response.content, prompt)
+        if errors.empty?
+          squawk(prompt, path, attempt, response, nil, reask:)
+          return produce(result, prompt)
+        end
+
+        repeated = reask && response.content == rejected.first
+        following = next_attempt(path, attempt, repeated)
+        error = invalid_output(response, errors, models.size, models.dup, prompt.role == :judge ? "Judge" : "LLM")
+        squawk(prompt, path, attempt, response, error, reask:, repeated:, final: following.nil?)
+        raise error unless following
+
+        rejected = [response.content, error.errors]
+        log_failure(prompt.role, attempt, path, rejection_reason(error.errors, response), following:, repeated:)
+        attempt = following
+      end
+    end
+
+    # One round of #samples: the pending samples' requests, sent concurrently, then each outcome recorded.
+    def round(pending, attempt)
+      jobs = pending.map do |sample|
+        step = sample.stage[attempt - 1]
+        message = sample_message(sample, step)
+        chat = sample.chat
+        attempts = sample.stage.size
+        reask = sample.reask
+        -> { @client.ask(chat, message, step, attempt:, attempts:, role: :sample, sample: sample.name, reask:) }
+      end
+      outcomes = @client.concurrently(jobs)
+      # A setup mistake (or anything that isn't a failed request) ends the call, whichever sample hit it.
+      fatal = outcomes.map(&:last).find { |error| error && !error.is_a?(LLMError) }
+      raise fatal if fatal
+
+      pending.zip(outcomes).each { |sample, outcome| record(sample, outcome, attempt) }
+    end
+
+    # The next message for a sample: a retry in its chat after invalid output, otherwise a fresh chat.
+    def sample_message(sample, step)
+      sample.reask = !sample.chat.nil?
+      return retry_message(sample.rejected.last) if sample.reask
+
+      sample.chat = start_chat(step, @prompt)
+      fresh_message(step, @prompt.input, sample.rejected)
+    end
+
+    def record(sample, (response, error), attempt)
+      stage = sample.stage
+      last = attempt == stage.size
+      role = "sample #{sample.name}"
+      if error
+        squawk(@prompt, stage, attempt, nil, error, sample:, reask: sample.reask)
+        raise error if last
+
+        sample.chat = nil
+        return log_failure(role, attempt, stage, error.message)
+      end
+
+      result, errors = @output_check.call(response.content, @prompt)
+      if errors.empty?
+        squawk(@prompt, stage, attempt, response, nil, sample:, reask: sample.reask)
+        return sample.result = produce(result, @prompt)
+      end
+
+      # A sample is one step (#samples refuses a stage that mixes them), so a repeat leaves nothing to move on to.
+      repeated = sample.reask && response.content == sample.rejected.first
+      invalid = invalid_output(response, errors, attempt, stage.first(attempt).map(&:model))
+      squawk(@prompt, stage, attempt, response, invalid, sample:, reask: sample.reask, repeated:,
+        final: last || repeated)
+      raise invalid if last || repeated
+
+      sample.rejected = [response.content, invalid.errors]
+      log_failure(role, attempt, stage, rejection_reason(invalid.errors, response))
+    end
+
+    # Remembers a result of the method's own output (a judge's verdict isn't one), so a harness can only return
+    # what a primitive produced.
+    def produce(result, prompt)
+      @produced[result] = true unless prompt.role == :judge
+      result
+    end
+
+    def start_chat(step, prompt)
+      @client.chat(step, instructions: prompt.instructions, schema: prompt.schema.llm_schema)
+    end
+
+    # Runs the observability hook, if any, with this attempt's raw output (nil when the call itself failed),
+    # the error that ended it (nil when it was accepted), and what was asked. role is nil for escalation
+    # attempts, :sample (with the sample's name) or :judge. reask is whether the attempt retried in the same chat,
+    # repeated whether it sent back the reply the attempt before it was rejected for, and final whether no attempt
+    # follows it.
+    def squawk(prompt, path, attempt, response, error, sample: nil, reask: false, repeated: false,
+      final: attempt == path.size)
+      hook = @definition.squawk
+      return unless hook && Squawk.report?(@definition.squawk_on, attempt, error, final)
+
+      step = path[attempt - 1]
+      metadata = {
+        label:, attempt:, attempts: path.size, final:, repeated:,
+        model: response&.model || step.model, provider: step.provider, params: step.request_params(reask),
+        input: prompt.input, usage: response&.usage, finish_reason: response&.finish_reason,
+        role: sample ? :sample : prompt.role, sample: sample&.name
+      }
+      Squawk.call(hook, output: response&.content, metadata:, error:)
+    end
+
+    # following is the attempt that comes next, which is not the next one when a repeated reply skips ahead.
+    def log_failure(role, attempt, path, reason, following: attempt + 1, repeated: false)
+      step = path[attempt - 1]
+      next_step = path[following - 1]
+      action = next_step == step ? "retrying" : "escalating to #{next_step.display_name}"
+      who = role ? "#{role} attempt" : "attempt"
+      reason = "it repeated the rejected reply: #{reason}" if repeated
+      log("#{who} #{attempt} of #{path.size} (#{step.display_name}) failed, #{action}: #{reason}")
+    end
+  end
+end

data/lib/squishling/schema.rb

@@ -26,17 +26,19 @@ module Squishling
       end
     end
 
-    # The payload handed to RubyLLM's `with_schema`: always strict, without LOCAL_ONLY_KEYWORDS.
+    # The payload handed to the adapter (Adapter::Request#schema): always strict, without LOCAL_ONLY_KEYWORDS.
     attr_reader :llm_schema
     # The full JSON Schema used for validation and result typing.
     attr_reader :json_schema
+    # The Data class results are built as, or nil when the schema isn't an object with properties.
+    attr_reader :result_class
 
     def initialize(raw)
       hash = self.class.jsonify(to_hash(raw))
       wrapped = wrapped?(hash)
       body = wrapped ? hash["schema"] : hash
 
-      # Same precedence RubyLLM uses: a top-level strict flag wins over one inside the schema.
+      # Same precedence RubyLLM 2.0 uses: a top-level strict flag wins over one inside the schema.
       strict = hash.key?("strict") ? hash["strict"] : body["strict"]
       raise ConfigurationError, "Squishling only supports strict output schemas (got strict: false)" if strict == false
 
@@ -48,22 +50,21 @@ module Squishling
         "strict" => true
       }.compact
       @validator = JSONSchemer.schema(@json_schema)
+      # Built now so a property name the result can't take fails here, not on the first response.
+      @result_class = Result.build(@json_schema)
     end
 
     def validate(data)
-      @validator.validate(data).map { |error| error["error"] }
+      errors = @validator.validate(data).map { |error| error["error"] }
+      # A raw Hash schema can leave out type: "object"; the typed result is built from a Hash, so anything else is
+      # invalid output rather than a NoMethodError out of the builder.
+      errors.empty? && result_class && !data.is_a?(Hash) ? ["root must be an object"] : errors
     end
 
     def build(data, squished:)
       result_class ? result_class.from_h(data, squished:) : data
     end
 
-    def result_class
-      return @result_class if defined?(@result_class)
-
-      @result_class = Result.build(@json_schema)
-    end
-
     private
 
     def to_hash(raw)

data/lib/squishling/squawk.rb

@@ -1,17 +1,47 @@
 # frozen_string_literal: true
 
 module Squishling
-  # The opt-in observability hook: a callable run after every LLM attempt with the attempt's raw output, so
-  # it can be sent somewhere (an error tracker, a tracing tool). It's the one sanctioned way for model output
-  # to leave the process; nothing runs unless one is configured.
+  # The opt-in observability hook: a callable run after every LLM attempt (or only the rejected and retried
+  # ones, see squawk_on:) with the attempt's raw output, so it can be sent somewhere (an error tracker, a
+  # tracing tool). It's the one sanctioned way for model output to leave the process; nothing runs unless one
+  # is configured.
   module Squawk
     KEYWORD_TYPES = %i[key keyreq].freeze
     POSITIONAL_TYPES = %i[req opt].freeze
     KEYWORD_CAPTURE_TYPES = %i[key keyreq keyrest].freeze
     KEYWORDS = %i[output metadata error].freeze
+    # The squawk_on: conditions, each answering "does the hook run for this attempt?" from the attempt's 1-based
+    # number (the one its metadata carries), its error (nil when accepted), and whether it is the last one.
+    CONDITIONS = {
+      all: ->(_attempt, _error, _final) { true },
+      rejections: ->(_attempt, error, _final) { !error.nil? },
+      retries: ->(attempt, error, _final) { !error.nil? || attempt > 1 },
+      final: ->(_attempt, _error, final) { final }
+    }.freeze
+    ON = CONDITIONS.keys.freeze
 
     module_function
 
+    # Which attempts reach the hook (squawk_on:): one condition or a list of them, or nil to inherit. :all is
+    # every attempt, :rejections only the ones that errored, :retries those plus any accepted attempt after the
+    # first, and :final the last attempt the escalation allows (metadata's final:), however it ended; an earlier
+    # accepted attempt is not final. A list runs the hook when any of them holds.
+    def validate_on(value, where)
+      return value if value.nil?
+
+      conditions = value.is_a?(Array) ? value : [value]
+      unless conditions.any? && conditions.all? { |condition| CONDITIONS.key?(condition) }
+        raise ConfigurationError,
+          "#{where} squawk_on must be one or a list of #{ON.map(&:inspect).join(', ')}, got #{value.inspect}"
+      end
+
+      value.is_a?(Array) ? value.dup.freeze : value
+    end
+
+    def report?(on, attempt, error, final)
+      Array(on).any? { |condition| CONDITIONS.fetch(condition).call(attempt, error, final) }
+    end
+
     # The hook itself, or nil/false to inherit/silence. Anything else must respond to call and take keywords
     # only, which is checked here so a mismatch fails at setup, not inside the first attempt.
     def validate(hook, where)

data/lib/squishling/tasks/squishling.rake

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Outside Rails, load it from your Rakefile: load "squishling/tasks/squishling.rake"
+unless Rake::Task.task_defined?("squishling:verify")
+  namespace :squishling do
+    desc "Verify every squished class without making a request (see Squishling.verify)"
+    task :verify do
+      # Only loaded classes are known, so load them all first. Rails defines :environment after loading its
+      # railties' tasks, so it can't be a prerequisite declared here.
+      Rake::Task[:environment].invoke if Rake::Task.task_defined?(:environment)
+      Rails.application.eager_load! if defined?(Rails) && Rails.respond_to?(:application) && Rails.application
+
+      report = Squishling.verify
+      puts report
+      abort unless report.ok?
+    end
+  end
+end

data/lib/squishling/testing/adapter.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # A Squishling adapter that answers from a script instead of a provider (see Script). Install it with
+    # Testing.install. It records every request (#requests) and groups them into the chats they belong to
+    # (#conversations), and it is safe to ask from the worker threads a harness's samples run on: everything shared
+    # is taken under one lock, and replies are chosen by (step, sample, attempt), never by arrival order.
+    class Adapter < Squishling::Adapter::Base
+      # One chat: its sample (nil unless a harness sample's) and the turns exchanged so far.
+      Conversation = Struct.new(:index, :sample, :transcript)
+
+      attr_reader :script
+
+      def initialize(script = Script.new)
+        super()
+        @script = script
+        @lock = Mutex.new
+        @requests = []
+        @conversations = []
+      end
+
+      def judgments?
+        true
+      end
+
+      # Every request so far, in the order they arrived (Testing::Request).
+      def requests
+        @lock.synchronize { @requests.dup }
+      end
+
+      # The requests of each chat, in order: a retry on the same step shares a chat, a new step starts one.
+      def conversations
+        requests.select(&:conversation).group_by(&:conversation).sort.map(&:last)
+      end
+
+      def requests_for(role: nil, sample: nil)
+        requests.select do |request|
+          (role.nil? || request.role == role) && (sample.nil? || request.sample == sample.to_s)
+        end
+      end
+
+      # The models the requests went to, in order (nil for a step that names none).
+      def models
+        requests.map(&:model)
+      end
+
+      # The first request's arguments: what left the process.
+      def arguments
+        requests.first&.arguments
+      end
+
+      def complete(request)
+        outcome = @lock.synchronize { answer(request) }
+        raise outcome if outcome.is_a?(Exception)
+
+        outcome
+      end
+
+      def judge(input, questions:, step:, label:)
+        outcome = @lock.synchronize { judged(input, questions, step, label) }
+        raise outcome if outcome.is_a?(Exception)
+
+        outcome
+      end
+
+      private
+
+      def answer(request)
+        call = call_of(request)
+        outcome = outcome_for(:chat, call, request.label, request.model)
+        conversation = conversation_for(request, call[:sample])
+        conversation.transcript.push(*turns(request, outcome)) unless outcome.is_a?(Exception)
+        record(request, call, outcome, conversation.index)
+        outcome
+      end
+
+      def judged(input, _questions, step, label)
+        call = { role: :judge, sample: nil, attempt: Observer.current&.attempt, attempts: Observer.current&.attempts,
+                 model: step.model, provider: step.provider }
+        outcome = outcome_for(:judgment, call, label, step.model)
+        payload = Schema.jsonify(input)
+        @requests << Request.new(label:, type: :judgment, role: :judge, sample: nil, attempt: call[:attempt],
+          attempts: call[:attempts], model: step.model, provider: step.provider, params: step.params,
+          system_prompt: nil, output_schema: nil, messages: [], conversation: nil, payload:, reply: outcome)
+        outcome
+      end
+
+      def outcome_for(kind, call, label, model)
+        rule = @script.rule_for(kind, call)
+        item = rule ? rule.take : Rule::EXHAUSTED
+        return unscripted(kind, call, label, model, rule) if item.equal?(Rule::EXHAUSTED)
+
+        resolve(item, label)
+      end
+
+      def resolve(item, label)
+        case item
+        when Failure then Failures.build(item.intent, label)
+        when Exception, Judgment then item
+        when Class then item <= Exception ? item.new : response(item)
+        when Reply then Squishling::Adapter::Response.new(content: item.content, model: item.model, tokens: item.usage)
+        else response(item)
+        end
+      end
+
+      def response(content)
+        Squishling::Adapter::Response.new(content:)
+      end
+
+      def unscripted(kind, call, label, model, rule)
+        aimed = call.compact.except(:attempts).map { |key, value| "#{key}: #{value.inspect}" }.join(", ")
+        why = if rule
+          "its script #{rule.empty? ? 'has no replies' : 'has no answers left (once)'}"
+        else
+          "no script covers it"
+        end
+        UnscriptedRequestError.new("#{label}: unscripted #{kind} request to #{model.inspect} (#{aimed}): #{why}. " \
+                                   "Scripted: #{@script.describe}")
+      end
+
+      # The role, sample, and attempt of the request: from the observer that wraps it, or, when the adapter was
+      # installed without one, what the request itself shows.
+      def call_of(request)
+        observed = Observer.current
+        base = { model: request.model, provider: request.provider }
+        if observed
+          return base.merge(role: observed.role, sample: observed.sample, attempt: observed.attempt,
+            attempts: observed.attempts)
+        end
+
+        base.merge(role: judge_schema?(request.schema) ? :judge : nil, sample: nil, attempt: nil, attempts: nil)
+      end
+
+      def judge_schema?(schema)
+        schema["schema"] == Schema.for(Judge::VERDICT_SCHEMA).llm_schema["schema"]
+      end
+
+      # A request that continues a chat sends the chat's transcript back, so it finds its conversation by it.
+      def conversation_for(request, sample)
+        history = request.messages[0...-1]
+        found = history.empty? ? nil : @conversations.reverse.find { |c| c.sample == sample && c.transcript == history }
+        found || Conversation.new(@conversations.size, sample, []).tap { |chat| @conversations << chat }
+      end
+
+      def turns(request, response)
+        [{ role: :user, content: request.messages.last.fetch(:content) }, { role: :assistant, content: response.text }]
+      end
+
+      def record(request, call, outcome, conversation)
+        @requests << Request.new(label: request.label, type: :chat, role: call[:role], sample: call[:sample]&.to_s,
+          attempt: call[:attempt], attempts: call[:attempts], model: request.model, provider: request.provider,
+          params: request.params, system_prompt: request.instructions, output_schema: request.schema["schema"],
+          messages: request.messages, conversation:, payload: Request.payload_of(request.messages), reply: outcome)
+      end
+    end
+
+    # The adapter behind Testing.forbid_provider_calls!: any request that reaches it raises ProviderCallError, which
+    # is not a Squishling::Error, so a squish_fallback can't absorb it. It remembers the calls that were attempted,
+    # for a test that rescued the error.
+    class ForbiddenAdapter < Squishling::Adapter::Base
+      def initialize
+        super
+        @lock = Mutex.new
+        @attempts = []
+      end
+
+      # The labels ("Klass#method") of the calls that reached it.
+      def attempts
+        @lock.synchronize { @attempts.dup }
+      end
+
+      def judgments?
+        true
+      end
+
+      def complete(request)
+        refuse(request.label)
+      end
+
+      def judge(_input, label:, **)
+        refuse(label)
+      end
+
+      private
+
+      def refuse(label)
+        @lock.synchronize { @attempts << label }
+        raise ProviderCallError, "#{label} reached the provider, but no LLM call was expected. Script it with " \
+                                 "Squishling::Testing.install, or fix the code under test"
+      end
+    end
+  end
+end

data/lib/squishling/testing/checks.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # The assertions, independent of any test framework: each returns a failure message, or nil when it holds. The
+    # Minitest and RSpec mixins turn a message into a failure in their own way.
+    module Checks
+      module_function
+
+      # The requests of a fake adapter (or an Array of them) sent exactly these arguments, as JSON would carry them.
+      def sent_arguments(source, expected, request: 0)
+        sent = requests_of(source)[request]
+        return "expected request ##{request} to have been sent, but only #{requests_of(source).size} were" unless sent
+
+        wanted = Schema.jsonify(expected)
+        return nil if sent.arguments == wanted
+
+        "expected the model to be sent arguments #{wanted.inspect}, but it was sent #{sent.arguments.inspect}"
+      end
+
+      def call_count(source, expected)
+        actual = requests_of(source).size
+        actual == expected ? nil : "expected #{expected} LLM call(s), but there were #{actual}"
+      end
+
+      # Runs the block with provider calls forbidden: a failure message when it tried to make one.
+      def no_llm_call(&)
+        attempts = Testing.no_llm_call(&)
+        return nil if attempts.empty?
+
+        "expected no LLM call, but #{attempts.uniq.join(', ')} reached the provider (#{attempts.size} request(s))"
+      end
+
+      def requests_of(source)
+        source.respond_to?(:requests) ? source.requests : Array(source)
+      end
+    end
+  end
+end