squishling 0.4.0 → 0.5.0

data/lib/squishling/squawk.rb

@@ -1,17 +1,47 @@
 # frozen_string_literal: true
 
 module Squishling
-  # The opt-in observability hook: a callable run after every LLM attempt with the attempt's raw output, so
-  # it can be sent somewhere (an error tracker, a tracing tool). It's the one sanctioned way for model output
-  # to leave the process; nothing runs unless one is configured.
+  # The opt-in observability hook: a callable run after every LLM attempt (or only the rejected and retried
+  # ones, see squawk_on:) with the attempt's raw output, so it can be sent somewhere (an error tracker, a
+  # tracing tool). It's the one sanctioned way for model output to leave the process; nothing runs unless one
+  # is configured.
   module Squawk
     KEYWORD_TYPES = %i[key keyreq].freeze
     POSITIONAL_TYPES = %i[req opt].freeze
     KEYWORD_CAPTURE_TYPES = %i[key keyreq keyrest].freeze
     KEYWORDS = %i[output metadata error].freeze
+    # The squawk_on: conditions, each answering "does the hook run for this attempt?" from the attempt's 1-based
+    # number (the one its metadata carries), its error (nil when accepted), and whether it is the last one.
+    CONDITIONS = {
+      all: ->(_attempt, _error, _final) { true },
+      rejections: ->(_attempt, error, _final) { !error.nil? },
+      retries: ->(attempt, error, _final) { !error.nil? || attempt > 1 },
+      final: ->(_attempt, _error, final) { final }
+    }.freeze
+    ON = CONDITIONS.keys.freeze
 
     module_function
 
+    # Which attempts reach the hook (squawk_on:): one condition or a list of them, or nil to inherit. :all is
+    # every attempt, :rejections only the ones that errored, :retries those plus any accepted attempt after the
+    # first, and :final the last attempt the escalation allows (metadata's final:), however it ended; an earlier
+    # accepted attempt is not final. A list runs the hook when any of them holds.
+    def validate_on(value, where)
+      return value if value.nil?
+
+      conditions = value.is_a?(Array) ? value : [value]
+      unless conditions.any? && conditions.all? { |condition| CONDITIONS.key?(condition) }
+        raise ConfigurationError,
+          "#{where} squawk_on must be one or a list of #{ON.map(&:inspect).join(', ')}, got #{value.inspect}"
+      end
+
+      value.is_a?(Array) ? value.dup.freeze : value
+    end
+
+    def report?(on, attempt, error, final)
+      Array(on).any? { |condition| CONDITIONS.fetch(condition).call(attempt, error, final) }
+    end
+
     # The hook itself, or nil/false to inherit/silence. Anything else must respond to call and take keywords
     # only, which is checked here so a mismatch fails at setup, not inside the first attempt.
     def validate(hook, where)

data/lib/squishling/tasks/squishling.rake

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Outside Rails, load it from your Rakefile: load "squishling/tasks/squishling.rake"
+unless Rake::Task.task_defined?("squishling:verify")
+  namespace :squishling do
+    desc "Verify every squished class without making a request (see Squishling.verify)"
+    task :verify do
+      # Only loaded classes are known, so load them all first. Rails defines :environment after loading its
+      # railties' tasks, so it can't be a prerequisite declared here.
+      Rake::Task[:environment].invoke if Rake::Task.task_defined?(:environment)
+      Rails.application.eager_load! if defined?(Rails) && Rails.respond_to?(:application) && Rails.application
+
+      report = Squishling.verify
+      puts report
+      abort unless report.ok?
+    end
+  end
+end

data/lib/squishling/testing/adapter.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # A Squishling adapter that answers from a script instead of a provider (see Script). Install it with
+    # Testing.install. It records every request (#requests) and groups them into the chats they belong to
+    # (#conversations), and it is safe to ask from the worker threads a harness's samples run on: everything shared
+    # is taken under one lock, and replies are chosen by (step, sample, attempt), never by arrival order.
+    class Adapter < Squishling::Adapter::Base
+      # One chat: its sample (nil unless a harness sample's) and the turns exchanged so far.
+      Conversation = Struct.new(:index, :sample, :transcript)
+
+      attr_reader :script
+
+      def initialize(script = Script.new)
+        super()
+        @script = script
+        @lock = Mutex.new
+        @requests = []
+        @conversations = []
+      end
+
+      def judgments?
+        true
+      end
+
+      # Every request so far, in the order they arrived (Testing::Request).
+      def requests
+        @lock.synchronize { @requests.dup }
+      end
+
+      # The requests of each chat, in order: a retry on the same step shares a chat, a new step starts one.
+      def conversations
+        requests.select(&:conversation).group_by(&:conversation).sort.map(&:last)
+      end
+
+      def requests_for(role: nil, sample: nil)
+        requests.select do |request|
+          (role.nil? || request.role == role) && (sample.nil? || request.sample == sample.to_s)
+        end
+      end
+
+      # The models the requests went to, in order (nil for a step that names none).
+      def models
+        requests.map(&:model)
+      end
+
+      # The first request's arguments: what left the process.
+      def arguments
+        requests.first&.arguments
+      end
+
+      def complete(request)
+        outcome = @lock.synchronize { answer(request) }
+        raise outcome if outcome.is_a?(Exception)
+
+        outcome
+      end
+
+      def judge(input, questions:, step:, label:)
+        outcome = @lock.synchronize { judged(input, questions, step, label) }
+        raise outcome if outcome.is_a?(Exception)
+
+        outcome
+      end
+
+      private
+
+      def answer(request)
+        call = call_of(request)
+        outcome = outcome_for(:chat, call, request.label, request.model)
+        conversation = conversation_for(request, call[:sample])
+        conversation.transcript.push(*turns(request, outcome)) unless outcome.is_a?(Exception)
+        record(request, call, outcome, conversation.index)
+        outcome
+      end
+
+      def judged(input, _questions, step, label)
+        call = { role: :judge, sample: nil, attempt: Observer.current&.attempt, attempts: Observer.current&.attempts,
+                 model: step.model, provider: step.provider }
+        outcome = outcome_for(:judgment, call, label, step.model)
+        payload = Schema.jsonify(input)
+        @requests << Request.new(label:, type: :judgment, role: :judge, sample: nil, attempt: call[:attempt],
+          attempts: call[:attempts], model: step.model, provider: step.provider, params: step.params,
+          system_prompt: nil, output_schema: nil, messages: [], conversation: nil, payload:, reply: outcome)
+        outcome
+      end
+
+      def outcome_for(kind, call, label, model)
+        rule = @script.rule_for(kind, call)
+        item = rule ? rule.take : Rule::EXHAUSTED
+        return unscripted(kind, call, label, model, rule) if item.equal?(Rule::EXHAUSTED)
+
+        resolve(item, label)
+      end
+
+      def resolve(item, label)
+        case item
+        when Failure then Failures.build(item.intent, label)
+        when Exception, Judgment then item
+        when Class then item <= Exception ? item.new : response(item)
+        when Reply then Squishling::Adapter::Response.new(content: item.content, model: item.model, tokens: item.usage)
+        else response(item)
+        end
+      end
+
+      def response(content)
+        Squishling::Adapter::Response.new(content:)
+      end
+
+      def unscripted(kind, call, label, model, rule)
+        aimed = call.compact.except(:attempts).map { |key, value| "#{key}: #{value.inspect}" }.join(", ")
+        why = if rule
+          "its script #{rule.empty? ? 'has no replies' : 'has no answers left (once)'}"
+        else
+          "no script covers it"
+        end
+        UnscriptedRequestError.new("#{label}: unscripted #{kind} request to #{model.inspect} (#{aimed}): #{why}. " \
+                                   "Scripted: #{@script.describe}")
+      end
+
+      # The role, sample, and attempt of the request: from the observer that wraps it, or, when the adapter was
+      # installed without one, what the request itself shows.
+      def call_of(request)
+        observed = Observer.current
+        base = { model: request.model, provider: request.provider }
+        if observed
+          return base.merge(role: observed.role, sample: observed.sample, attempt: observed.attempt,
+            attempts: observed.attempts)
+        end
+
+        base.merge(role: judge_schema?(request.schema) ? :judge : nil, sample: nil, attempt: nil, attempts: nil)
+      end
+
+      def judge_schema?(schema)
+        schema["schema"] == Schema.for(Judge::VERDICT_SCHEMA).llm_schema["schema"]
+      end
+
+      # A request that continues a chat sends the chat's transcript back, so it finds its conversation by it.
+      def conversation_for(request, sample)
+        history = request.messages[0...-1]
+        found = history.empty? ? nil : @conversations.reverse.find { |c| c.sample == sample && c.transcript == history }
+        found || Conversation.new(@conversations.size, sample, []).tap { |chat| @conversations << chat }
+      end
+
+      def turns(request, response)
+        [{ role: :user, content: request.messages.last.fetch(:content) }, { role: :assistant, content: response.text }]
+      end
+
+      def record(request, call, outcome, conversation)
+        @requests << Request.new(label: request.label, type: :chat, role: call[:role], sample: call[:sample]&.to_s,
+          attempt: call[:attempt], attempts: call[:attempts], model: request.model, provider: request.provider,
+          params: request.params, system_prompt: request.instructions, output_schema: request.schema["schema"],
+          messages: request.messages, conversation:, payload: Request.payload_of(request.messages), reply: outcome)
+      end
+    end
+
+    # The adapter behind Testing.forbid_provider_calls!: any request that reaches it raises ProviderCallError, which
+    # is not a Squishling::Error, so a squish_fallback can't absorb it. It remembers the calls that were attempted,
+    # for a test that rescued the error.
+    class ForbiddenAdapter < Squishling::Adapter::Base
+      def initialize
+        super
+        @lock = Mutex.new
+        @attempts = []
+      end
+
+      # The labels ("Klass#method") of the calls that reached it.
+      def attempts
+        @lock.synchronize { @attempts.dup }
+      end
+
+      def judgments?
+        true
+      end
+
+      def complete(request)
+        refuse(request.label)
+      end
+
+      def judge(_input, label:, **)
+        refuse(label)
+      end
+
+      private
+
+      def refuse(label)
+        @lock.synchronize { @attempts << label }
+        raise ProviderCallError, "#{label} reached the provider, but no LLM call was expected. Script it with " \
+                                 "Squishling::Testing.install, or fix the code under test"
+      end
+    end
+  end
+end

data/lib/squishling/testing/checks.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # The assertions, independent of any test framework: each returns a failure message, or nil when it holds. The
+    # Minitest and RSpec mixins turn a message into a failure in their own way.
+    module Checks
+      module_function
+
+      # The requests of a fake adapter (or an Array of them) sent exactly these arguments, as JSON would carry them.
+      def sent_arguments(source, expected, request: 0)
+        sent = requests_of(source)[request]
+        return "expected request ##{request} to have been sent, but only #{requests_of(source).size} were" unless sent
+
+        wanted = Schema.jsonify(expected)
+        return nil if sent.arguments == wanted
+
+        "expected the model to be sent arguments #{wanted.inspect}, but it was sent #{sent.arguments.inspect}"
+      end
+
+      def call_count(source, expected)
+        actual = requests_of(source).size
+        actual == expected ? nil : "expected #{expected} LLM call(s), but there were #{actual}"
+      end
+
+      # Runs the block with provider calls forbidden: a failure message when it tried to make one.
+      def no_llm_call(&)
+        attempts = Testing.no_llm_call(&)
+        return nil if attempts.empty?
+
+        "expected no LLM call, but #{attempts.uniq.join(', ')} reached the provider (#{attempts.size} request(s))"
+      end
+
+      def requests_of(source)
+        source.respond_to?(:requests) ? source.requests : Array(source)
+      end
+    end
+  end
+end

data/lib/squishling/testing/direct.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # Runs the parts of a squished method that don't need a model, for unit tests: the output checks, the fallback,
+    # and the prompt. Each takes the method's arguments as args: and keywords (as the method is called), and a
+    # receiver: for a class whose constructor takes arguments.
+    module Direct
+      class << self
+        # The messages squish_validate and the output schema reject this reply with ([] when it would be accepted).
+        def validate(klass, reply, method: :call, receiver: nil, args: [], **kwargs)
+          definition = klass.squishling_definition(method)
+          schema = definition.schema or raise SetupError, "#{definition.label} has no output_schema"
+          inputs = definition.bind_arguments(args, kwargs)
+          prompt = Invoker::Prompt.new(instructions: nil, schema:, input: nil, role: nil)
+          OutputCheck.new(definition, receiver || klass.new, inputs).call(reply, prompt).last.map(&:message)
+        end
+
+        # What squish_fallback returns for this error, typed like any result of the method.
+        def fallback_for(klass, error, method: :call, receiver: nil, args: [], **kwargs)
+          definition = klass.squishling_definition(method)
+          handler = definition.fallback or raise SetupError, "#{definition.label} has no squish_fallback"
+          inputs = definition.bind_arguments(args, kwargs)
+          definition.coerce((receiver || klass.new).instance_exec(error, **inputs, &handler))
+        end
+
+        # The first request a call would make, without making it: { system_prompt:, user_message:, output_schema:,
+        # step: } (the step is the ModelPath::Step with the model, provider and params; rotation isn't advanced).
+        def preview(klass, method: :call, receiver: nil, args: [], **kwargs)
+          definition = klass.squishling_definition(method)
+          receiver ||= klass.new
+          inputs = definition.bind_arguments(args, kwargs)
+          prompt = Invoker.new(definition, receiver, inputs).prompt
+          { system_prompt: prompt.instructions, user_message: prompt.input,
+            output_schema: prompt.schema.llm_schema["schema"],
+            step: definition.escalation_path(receiver, inputs, advance: false).first }
+        end
+      end
+    end
+  end
+end

data/lib/squishling/testing/errors.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # What the test helpers raise. None of these is a Squishling::Error, so squish_fallback, which absorbs
+    # InvalidOutputError and LLMError, can never hide a test setup mistake.
+    class Error < StandardError; end
+
+    # A request reached the test adapter and no script covers it.
+    class UnscriptedRequestError < Error; end
+
+    # A request reached the provider seam while provider calls were forbidden (see Testing.forbid_provider_calls!).
+    class ProviderCallError < Error; end
+
+    # A stub, a fixture, or a direct check was asked for something the method's schema doesn't allow.
+    class SetupError < Error; end
+  end
+end

data/lib/squishling/testing/fixtures.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # Replies derived from an output schema, so a test that doesn't care about the content stays valid when the schema
+    # changes, and a test of "out of range gets re-asked" doesn't hand-write a bad payload. Replies are Hashes with
+    # string keys, as JSON would carry them, and are always checked against the schema before they are returned.
+    module Fixtures
+      FORMATS = {
+        "date-time" => "2024-01-01T00:00:00Z", "date" => "2024-01-01", "time" => "00:00:00Z",
+        "email" => "user@example.com", "uri" => "https://example.com", "uuid" => "00000000-0000-4000-8000-000000000000"
+      }.freeze
+      MAX_DEPTH = 8
+      WRONG_VALUES = ["not valid", -1_000_000_007, [], {}, true, nil].freeze
+
+      class << self
+        # A reply that satisfies the schema of a squished class's method, a Squishling::Schema, or a schema Hash.
+        # overrides replace top-level attributes.
+        def valid_reply_for(source, method: :call, **overrides)
+          schema = schema_of(source, method)
+          reply = Generator.new(schema.json_schema).value
+          reply = reply.merge(overrides.transform_keys(&:to_s)) if reply.is_a?(Hash)
+          verify!(schema, reply, "valid")
+        end
+
+        # A valid reply with one top-level attribute (violating:) broken: out of its enum or range, the wrong type,
+        # null, or missing. how: (:enum, :range, :type, :null, :missing) picks the break; by default the first that
+        # the schema rejects. Overrides are applied to the valid reply first.
+        def invalid_reply_for(source, violating:, how: nil, method: :call, **overrides)
+          schema = schema_of(source, method)
+          base = valid_reply_for(schema, **overrides)
+          key = violating.to_s
+          raise SetupError, "the schema has no property #{key.inspect} to violate" unless schema.json_schema.fetch(
+            "properties", {}
+          ).key?(key)
+
+          candidates = breaks(schema.json_schema, key, how)
+          broken = candidates.lazy.map { |value| mutate(base, key, value) }.find { |reply| schema.validate(reply).any? }
+          broken or raise SetupError,
+            "couldn't find a value of #{key.inspect} that the schema rejects#{" (how: #{how.inspect})" if how}"
+        end
+
+        private
+
+        def schema_of(source, method)
+          case source
+          when Schema then source
+          when Class then source.squishling_definition(method).schema or raise SetupError,
+            "#{source} has no output_schema"
+          else Schema.for(source)
+          end
+        end
+
+        def verify!(schema, reply, kind)
+          errors = schema.validate(reply)
+          return reply if errors.empty?
+
+          raise SetupError,
+            "couldn't build a #{kind} reply (#{errors.join('; ')}); pass the attributes it needs as overrides"
+        end
+
+        def mutate(base, key, value)
+          value.equal?(:missing) ? base.except(key) : base.merge(key => value)
+        end
+
+        def breaks(json_schema, key, how)
+          node = Generator.new(json_schema).node_for(json_schema.fetch("properties").fetch(key))
+          all = { enum: enum_breaks(node), range: range_breaks(node), type: WRONG_VALUES.compact, null: [nil],
+                  missing: [:missing] }
+          return all.fetch(how) { raise ArgumentError, "how: is one of #{all.keys.map(&:inspect).join(', ')}" } if how
+
+          all.values.flatten(1)
+        end
+
+        def enum_breaks(node)
+          values = node["enum"] ? ["not in the enum"] : []
+          node.key?("const") ? values + ["not the const"] : values
+        end
+
+        def range_breaks(node)
+          bounds = node.slice("minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum", "minLength", "maxLength",
+            "minItems", "maxItems")
+          bounds.flat_map { |name, limit| beyond(name, limit) }
+        end
+
+        def beyond(name, limit)
+          case name
+          when "minimum", "exclusiveMinimum" then [limit - 1]
+          when "maximum", "exclusiveMaximum" then [limit + 1]
+          when "minLength" then ["x" * [limit - 1, 0].max]
+          when "maxLength" then ["x" * (limit + 1)]
+          when "minItems" then [[nil] * [limit - 1, 0].max]
+          when "maxItems" then [[nil] * (limit + 1)]
+          end
+        end
+      end
+
+      # Builds one value that satisfies a JSON schema (see Fixtures).
+      class Generator
+        def initialize(root)
+          @root = root
+          @expanding = []
+        end
+
+        # The value for a node. A $ref being expanded is remembered, so a recursive schema (a node whose child is
+        # the same node or null) ends at the null instead of nesting forever.
+        def value(node = @root, depth = 0)
+          ref = node["$ref"]
+          @expanding.push(ref) if ref
+          generate(node_for(node), depth)
+        ensure
+          @expanding.pop if ref
+        end
+
+        # The schema node with a root-relative $ref followed.
+        def node_for(node)
+          ref = node["$ref"]
+          return node unless ref
+
+          path = ref.delete_prefix("#").split("/").reject(&:empty?)
+          node.except("$ref").merge(path.reduce(@root) do |memo, part|
+            memo.fetch(part.gsub("~1", "/").gsub("~0", "~"))
+          end)
+        end
+
+        private
+
+        def generate(node, depth)
+          return node["const"] if node.key?("const")
+          return node["enum"].first if node["enum"]
+          return from_branch(node, depth) if node["anyOf"] || node["oneOf"]
+          return from_all_of(node, depth) if node["allOf"]
+
+          typed(node, depth)
+        end
+
+        def from_branch(node, depth)
+          branches = node["anyOf"] || node["oneOf"]
+          open = branches.reject { |branch| branch["$ref"] && @expanding.include?(branch["$ref"]) }
+          chosen = open.find { |branch| branch["type"] != "null" } || open.first || branches.first
+          value(chosen, depth + 1)
+        end
+
+        def from_all_of(node, depth)
+          parts = node["allOf"].map { |part| value(part, depth + 1) }.grep(Hash)
+          parts.reduce({}) { |memo, part| memo.merge(part) }
+        end
+
+        def typed(node, depth)
+          case primary_type(node)
+          when "object" then object(node, depth)
+          when "array" then array(node, depth)
+          when "string" then string(node)
+          when "integer", "number" then number(node)
+          when "boolean" then true
+          end
+        end
+
+        def primary_type(node)
+          types = Array(node["type"])
+          types.find { |type| type != "null" } || (types.first if types.any?) || inferred_type(node)
+        end
+
+        def inferred_type(node)
+          return "object" if node["properties"]
+
+          "array" if node["items"]
+        end
+
+        def object(node, depth)
+          return {} if depth > MAX_DEPTH
+
+          (node["properties"] || {}).transform_values { |property| value(property, depth + 1) }
+        end
+
+        def array(node, depth)
+          return [] if depth > MAX_DEPTH || !node["items"]
+
+          Array.new(node["minItems"].to_i) { value(node["items"], depth + 1) }
+        end
+
+        def string(node)
+          text = FORMATS.fetch(node["format"], "example")
+          text = text.ljust(node["minLength"], "x") if node["minLength"]
+          node["maxLength"] ? text[0, node["maxLength"]] : text
+        end
+
+        def number(node)
+          number = clamp_up(0, node["minimum"], node["exclusiveMinimum"])
+          number = clamp_down(number, node["maximum"], node["exclusiveMaximum"])
+          primary_type(node) == "integer" ? number.ceil : number
+        end
+
+        def clamp_up(number, minimum, exclusive)
+          number = minimum if minimum && number < minimum
+          exclusive && number <= exclusive ? exclusive + 1 : number
+        end
+
+        def clamp_down(number, maximum, exclusive)
+          number = maximum if maximum && number > maximum
+          exclusive && number >= exclusive ? exclusive - 1 : number
+        end
+      end
+    end
+  end
+end

data/lib/squishling/testing/helpers.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # The test-facing helpers, shared by the Minitest and RSpec mixins. A mixin supplies #squishling_failure(message),
+    # which fails the current test.
+    module Helpers
+      # Installs a scripted adapter and returns it. The block scripts it (see Script); the adapter's #requests,
+      # #conversations and #arguments show what was sent.
+      def fake_squishling(&)
+        Testing.install(&)
+      end
+
+      # Any LLM request raises until a test installs a fake, naming the squished method that made it.
+      def forbid_provider_calls!
+        Testing.forbid_provider_calls!
+      end
+
+      def assert_sent_arguments(source, expected, request: 0)
+        failure = Checks.sent_arguments(source, expected, request:)
+        squishling_failure(failure) if failure
+      end
+
+      def assert_llm_calls(source, expected)
+        failure = Checks.call_count(source, expected)
+        squishling_failure(failure) if failure
+      end
+
+      def assert_no_llm_call(&)
+        failure = Checks.no_llm_call(&)
+        squishling_failure(failure) if failure
+      end
+
+      def stub_squishling_result(klass, attrs = nil, **, &)
+        Testing.stub_result(klass, attrs, **, &)
+      end
+
+      def stub_squishling_error(klass, kind, **, &)
+        Testing.stub_error(klass, kind, **, &)
+      end
+
+      def squishling_validate(klass, reply, **inputs)
+        Testing.validate(klass, reply, **inputs)
+      end
+
+      def squishling_fallback_for(klass, error, **inputs)
+        Testing.fallback_for(klass, error, **inputs)
+      end
+
+      def squishling_preview(klass, **inputs)
+        Testing.preview(klass, **inputs)
+      end
+
+      def valid_reply_for(source, **overrides)
+        Testing.valid_reply_for(source, **overrides)
+      end
+
+      def invalid_reply_for(source, **)
+        Testing.invalid_reply_for(source, **)
+      end
+
+      def capture_squawk
+        Testing.capture_squawk
+      end
+
+      def with_squawk(capture = capture_squawk, &)
+        Testing.with_squawk(capture, &)
+      end
+    end
+  end
+end

data/lib/squishling/testing/minitest.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # The helpers for Minitest: include it in a test case (or ActiveSupport::TestCase). What it installs (the
+    # adapter, observer, and stubs) is put back after each test.
+    #
+    #   class RegexInferrerTest < Minitest::Test
+    #     include Squishling::Testing::Minitest
+    #
+    #     def test_retries_a_bad_regex
+    #       fake = fake_squishling { |llm| llm.step(:any).reply({ regex: "[bad(" }, { regex: "Run.*" }) }
+    #       assert_equal "Run.*", RegexInferrer.call(sample_turns: turns).regex
+    #       assert_sent_arguments fake, { sample_turns: turns }
+    #     end
+    #   end
+    module Minitest
+      include Helpers
+
+      def after_teardown
+        Testing.reset!
+        super
+      end
+
+      private
+
+      def squishling_failure(message)
+        flunk message
+      end
+    end
+  end
+end