squishling 0.4.0 → 0.5.0

data/lib/squishling/testing/observer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # Tells the test adapter which request it is answering. An Adapter::Request carries no role, sample, or attempt,
+    # but the Observer::Call that wraps it does, on the thread that makes the request: this observer keeps it where
+    # Adapter#complete can read it, and passes the request on to any observer the app configured.
+    class Observer
+      KEY = :squishling_testing_call
+
+      # The call being made on this thread, or nil outside a request.
+      def self.current
+        Thread.current[KEY]
+      end
+
+      attr_reader :previous
+
+      def initialize(previous = nil)
+        @previous = previous
+      end
+
+      def around(call, &)
+        outer = Thread.current[KEY]
+        Thread.current[KEY] = call
+        @previous ? @previous.around(call, &) : yield
+      ensure
+        Thread.current[KEY] = outer
+      end
+    end
+  end
+end

data/lib/squishling/testing/reply.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # A scripted answer with more than content: the model id the provider reports and its token counts, so the
+    # squawk and observer usage paths are covered. usage is a Hash such as { input_tokens: 12, output_tokens: 30 }.
+    class Reply
+      attr_reader :content, :model, :usage
+
+      def initialize(content = nil, model: nil, usage: nil)
+        @content = content
+        @model = model
+        @usage = usage
+      end
+    end
+
+    # A scripted provider failure, named by what went wrong (see Failures).
+    Failure = Data.define(:intent)
+
+    # The answer to the "winner" question of a System One judgment.
+    Answer = Struct.new(:choice, :probabilities)
+
+    # What Adapter#judge returns: answers by question name, like RubyLLM's Judgment.
+    Judgment = Struct.new(:answers) do
+      def [](name)
+        answers[name]
+      end
+    end
+
+    # The provider failures a script can name by intent rather than by the error class of any one SDK. Each becomes
+    # the Squishling error an adapter raises for it: LLMError for what is worth another attempt, ConfigurationError
+    # for what is not. Messages carry the failure and its status, like the real adapters' do.
+    module Failures
+      # intent => [error class, what happened, HTTP status or nil]
+      INTENTS = {
+        rate_limit: [LLMError, "rate limited", 429],
+        server_error: [LLMError, "server error", 500],
+        overloaded: [LLMError, "provider overloaded", 529],
+        timeout: [LLMError, "request timed out", nil],
+        context_length: [LLMError, "context length exceeded", 400],
+        bad_request: [ConfigurationError, "bad request", 400],
+        unauthorized: [ConfigurationError, "unauthorized", 401],
+        forbidden: [ConfigurationError, "forbidden", 403]
+      }.freeze
+
+      module_function
+
+      def check!(intent)
+        return intent if INTENTS.key?(intent)
+
+        raise ArgumentError, "unknown failure #{intent.inspect} (one of #{INTENTS.keys.map(&:inspect).join(', ')})"
+      end
+
+      def build(intent, label)
+        klass, text, status = INTENTS.fetch(intent)
+        klass.new("#{label}: #{text}#{" (HTTP #{status})" if status}")
+      end
+    end
+  end
+end

data/lib/squishling/testing/request.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # Run's follow-up messages (a private constant of Run): the helpers read what was sent through their parsers.
+    MESSAGES = Run.const_get(:Messages)
+    private_constant :MESSAGES
+
+    # One provider request as the test adapter saw it: what left the process, and what came back.
+    #
+    # arguments and context are the method's arguments and the squish_context / squish! context values the model was
+    # sent. For a judge they are those of the call being judged, and candidates holds the two samples. messages is
+    # the conversation as sent (empty for a System One judgment, whose payload is the input). conversation is the
+    # index of the chat in Adapter#conversations (nil for a judgment): a retry is the same conversation, an
+    # escalation a new one. reply is the Adapter::Response, or the error raised instead.
+    Request = Data.define(:label, :type, :role, :sample, :attempt, :attempts, :model, :provider, :params,
+      :system_prompt, :output_schema, :messages, :conversation, :payload, :reply) do
+      def arguments
+        input["arguments"]
+      end
+
+      def context
+        input["context"]
+      end
+
+      def candidates
+        payload["candidates"]
+      end
+
+      # The validation messages this request carried back to the model: a retry in the same conversation, or an
+      # escalation to the next step. Empty for a first attempt.
+      def errors
+        (retry_errors || escalation&.errors) || []
+      end
+
+      # The rejected output forwarded from the previous step (text, capped), or nil when none was.
+      def forwarded_rejected
+        escalation&.rejected
+      end
+
+      def failed?
+        reply.is_a?(Exception)
+      end
+
+      def judge?
+        role == :judge
+      end
+
+      # The JSON payload of the first user message, parsed (an escalation message wraps it in its own text).
+      def self.payload_of(messages)
+        text = messages.first.fetch(:content)
+        JSON.parse(MESSAGES.parse_escalation(text)&.input || text)
+      rescue JSON::ParserError
+        {}
+      end
+
+      private
+
+      # The method's arguments and context: the payload itself, or inside a judge's "input".
+      def input
+        judge? ? payload.fetch("input", {}) : payload
+      end
+
+      def retry_errors
+        text = messages.last&.fetch(:content, nil)
+        messages.size > 1 && text ? MESSAGES.parse_retry(text) : nil
+      end
+
+      def escalation
+        text = messages.first&.fetch(:content, nil)
+        text && MESSAGES.parse_escalation(text)
+      end
+    end
+  end
+end

data/lib/squishling/testing/rspec.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # The helpers for RSpec (no rspec-mocks needed): `config.include Squishling::Testing::RSpec`, or include it in a
+    # group. What it installs (the adapter, observer, and stubs) is put back after each example.
+    #
+    #   RSpec.describe RegexInferrer do
+    #     it "retries a bad regex" do
+    #       fake = fake_squishling { |llm| llm.step(:any).reply({ regex: "[bad(" }, { regex: "Run.*" }) }
+    #       expect(described_class.call(sample_turns: turns).regex).to eq("Run.*")
+    #       assert_sent_arguments fake, { sample_turns: turns }
+    #     end
+    #   end
+    module RSpec
+      include Helpers
+
+      def self.included(group)
+        super
+        group.after { Testing.reset! }
+      end
+
+      private
+
+      def squishling_failure(message)
+        raise ::RSpec::Expectations::ExpectationNotMetError, message
+      end
+    end
+  end
+end

data/lib/squishling/testing/script.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # One scripted line: the requests it answers and what they get, in order. The last answer repeats unless the
+    # rule is #once. Build them through Script (the block Testing.install yields).
+    class Rule
+      # What a rule answers. nil means any. kind is :chat (Adapter#complete) or :judgment (Adapter#judge); role is
+      # nil (an escalation attempt, or any role when the rule names none), :sample, or :judge.
+      Selector = Data.define(:kind, :role, :sample, :model, :provider, :attempt)
+
+      # Rule#take when nothing is left to answer with.
+      EXHAUSTED = Object.new.freeze
+
+      attr_reader :selector
+
+      def initialize(selector)
+        @selector = selector
+        @replies = []
+        @taken = 0
+        @once = false
+      end
+
+      # Answers, in order: a Hash (the model's answer), a String (raw text, malformed or fenced JSON included), nil
+      # (an empty response), a Reply (with a model id and token usage), or an Exception (raised as is).
+      def reply(*replies)
+        @replies.concat(replies)
+        self
+      end
+
+      # A provider failure named by intent (see Failures::INTENTS).
+      def fail(intent)
+        @replies << Failure.new(Failures.check!(intent))
+        self
+      end
+
+      # Running out of answers is an error instead of repeating the last one.
+      def once
+        @once = true
+        self
+      end
+
+      def matches?(kind, call)
+        selector.kind == kind && fits?(selector.role, call[:role]) && fits?(selector.sample, call[:sample]) &&
+          fits?(selector.model, call[:model]) && fits?(selector.provider, call[:provider]) &&
+          fits?(selector.attempt, call[:attempt])
+      end
+
+      # How narrowly the rule aims, lowest first: rules for a sample or the judge, then those naming the most of a
+      # model, a provider, and an attempt, then the rest.
+      def specificity
+        return 0 if selector.sample || selector.role
+
+        4 - [selector.model, selector.provider, selector.attempt].count { |named| !named.nil? }
+      end
+
+      # The next answer, or EXHAUSTED. Called with the script's lock held.
+      def take
+        return EXHAUSTED if @replies.empty? || (@once && @taken >= @replies.size)
+
+        @replies[[@taken, @replies.size - 1].min].tap { @taken += 1 }
+      end
+
+      def empty?
+        @replies.empty?
+      end
+
+      def describe
+        aimed = selector.to_h.compact.except(:kind).map { |key, value| "#{key}: #{value.inspect}" }
+        "#{selector.kind} #{aimed.empty? ? 'any' : aimed.join(', ')}"
+      end
+
+      private
+
+      def fits?(wanted, actual)
+        wanted.nil? || wanted.to_s == actual.to_s
+      end
+    end
+
+    # The judge's rules: a chat judge answers a verdict, a System One judge a judgment with a confidence.
+    class JudgeRules
+      CHOICES = %i[a b neither].freeze
+
+      def initialize(chat, judgment)
+        @chat = chat
+        @judgment = judgment
+      end
+
+      # A chat judge's strict verdict (:a, :b, or :neither) and its reason.
+      def verdict(choice, reason: "scripted verdict")
+        @chat.reply({ "verdict" => check(choice).to_s, "reason" => reason })
+        self
+      end
+
+      # A System One judgment (type: :judgment) choosing :a, :b, or :neither with that probability; the rest of the
+      # probability is split between the other two choices. Pass an Exception to raise it instead.
+      def judgment(choice, confidence: 0.95)
+        unless choice.is_a?(Exception) || confidence.between?(0, 1)
+          raise ArgumentError, "confidence is a probability from 0 to 1, got #{confidence.inspect}"
+        end
+
+        @judgment.reply(choice.is_a?(Exception) ? choice : answer(check(choice), confidence))
+        self
+      end
+
+      # Anything a chat judge's request can answer with (see Rule#reply): off-schema verdicts, raw text, ...
+      def reply(*replies)
+        @chat.reply(*replies)
+        self
+      end
+
+      def fail(intent)
+        @chat.fail(intent)
+        self
+      end
+
+      def fail_judgment(intent)
+        @judgment.fail(intent)
+        self
+      end
+
+      def once
+        @chat.once
+        @judgment.once
+        self
+      end
+
+      private
+
+      def check(choice)
+        return choice.to_sym if CHOICES.include?(choice.to_s.to_sym)
+
+        raise ArgumentError, "a judge choice is one of #{CHOICES.map(&:inspect).join(', ')}, got #{choice.inspect}"
+      end
+
+      def answer(choice, confidence)
+        others = (1 - confidence) / 2.0
+        probabilities = CHOICES.to_h { |key| [key, key == choice ? confidence.to_f : others] }
+        Judgment.new({ winner: Answer.new(choice, probabilities) })
+      end
+    end
+
+    # What a test says the model will do. Rules are matched most specific first (a sample or the judge, then a
+    # named model, then step(:any)), and by declaration order within that.
+    #
+    #   llm.step("flash").reply({ score: 99 }, { score: 5 })   # garbage, then an answer
+    #   llm.step("pro").fail(:rate_limit)
+    #   llm.sample("a").reply(valid_a)
+    #   llm.judge.verdict(:a)
+    class Script
+      def initialize
+        @rules = []
+      end
+
+      # Escalation attempts (and any other chat request no sample or judge rule claims) on a model, a provider, or
+      # :any model; attempt: narrows it to one attempt of the escalation (counting from 1).
+      def step(model = :any, provider: nil, attempt: nil)
+        add(:chat, model: model == :any ? nil : model, provider:, attempt:)
+      end
+
+      # A harness sample's requests ("a", "b", ...). Each sample has its own queue, so the order the sample threads
+      # happen to run in can't change what a sample gets.
+      def sample(name)
+        add(:chat, role: :sample, sample: name.to_s)
+      end
+
+      # The judge of :judged_squishsum, chat or System One.
+      def judge
+        @judge ||= JudgeRules.new(add(:chat, role: :judge), add(:judgment, role: :judge))
+      end
+
+      # The rule that answers a request of this kind, or nil. Called with the lock held (see Adapter).
+      def rule_for(kind, call)
+        candidates = @rules.each_with_index.select { |rule, _| rule.matches?(kind, call) }
+        candidates.min_by { |rule, index| [rule.specificity, index] }&.first
+      end
+
+      def describe
+        @rules.empty? ? "nothing" : @rules.map(&:describe).join("; ")
+      end
+
+      private
+
+      def add(kind, role: nil, sample: nil, model: nil, provider: nil, attempt: nil)
+        Rule.new(Rule::Selector.new(kind:, role:, sample:, model:, provider:, attempt:)).tap { |rule| @rules << rule }
+      end
+    end
+  end
+end

data/lib/squishling/testing/state.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # What the helpers changed in Squishling.config (the adapter and observer), remembered so Testing.reset! puts
+    # back exactly that: not an app's other settings (which Squishling.reset_config! would also clear), and not a
+    # value a test assigned itself after the helpers did.
+    module State
+      LOCK = Mutex.new
+      private_constant :LOCK
+
+      @entries = {} # setting => [what it was before the helpers, what the helpers set it to]
+      @config = nil
+
+      class << self
+        # Sets a Squishling.config setting, remembering what it was before the first change.
+        def set(setting, value)
+          LOCK.synchronize do
+            config = Squishling.config
+            @entries = {} unless config.equal?(@config) # a replaced config is already pristine
+            @config = config
+            original = original_of(config, setting)
+            config.public_send(:"#{setting}=", value)
+            @entries[setting] = [original, value]
+          end
+        end
+
+        # Puts back the settings still holding what the helpers set; leaves a config that has been replaced alone.
+        def restore!
+          config, entries = LOCK.synchronize do
+            [@config, @entries].tap do
+              @entries = {}
+              @config = nil
+            end
+          end
+          return unless config.equal?(Squishling.config)
+
+          entries.each do |setting, (original, installed)|
+            config.public_send(:"#{setting}=", original) if config.public_send(setting).equal?(installed)
+          end
+        end
+
+        private
+
+        # What to put back: the value before the helpers' first change, unless something else was assigned since.
+        def original_of(config, setting)
+          current = config.public_send(setting)
+          entry = @entries[setting]
+          entry && current.equal?(entry.last) ? entry.first : current
+        end
+      end
+    end
+
+    # What a test asked squawk to record: every attempt's raw output, error, and metadata (role, sample, attempt,
+    # final, usage, ...). Use Testing.with_squawk, which puts the previous hook back.
+    class SquawkCapture
+      Call = Data.define(:output, :error, :metadata)
+
+      def initialize
+        @lock = Mutex.new
+        @calls = []
+      end
+
+      def call(output: nil, error: nil, metadata: nil)
+        @lock.synchronize { @calls << Call.new(output:, error:, metadata:) }
+      end
+
+      def calls
+        @lock.synchronize { @calls.dup }
+      end
+    end
+  end
+end

data/lib/squishling/testing/stubs.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # Replaces a squished method for the tests of its callers: the method returns (or raises) what the test says and
+    # never reaches Squishling, but what it returns is a real typed result, validated against the method's output
+    # schema, so the stand-in can't drift from the real result type. Cleared by Testing.reset!.
+    module Stubs
+      LOCK = Mutex.new
+      private_constant :LOCK
+
+      # [class, method name] => callable producing the result, or raising the error
+      @active = {}
+      # [class, method name] => the module prepended to the class. A prepended module can't be removed, so it stays
+      # (inert once its stub is cleared) and keeps its class alive: stub the classes of a suite, not one per test.
+      @hooks = {}
+
+      ERRORS = {
+        invalid_output: :invalid_output_error, llm_error: :llm_error, configuration: :configuration_error,
+        disagreement: :disagreement_error
+      }.freeze
+
+      class << self
+        # A result of the method's schema, with these attributes (squished: false for one the Ruby path returned).
+        def result(klass, attrs, method: :call, squished: true, &)
+          definition = klass.squishling_definition(method)
+          built = typed(definition, attrs, squished:)
+          activate(klass, method, -> { built }, &)
+        end
+
+        # The method raises this kind of error: :invalid_output (errors:, raw:, attempts:, models:), :llm_error
+        # (message:), :configuration (message:), or :disagreement (candidates:, verdict:, reason:, models:).
+        def error(klass, kind, method: :call, **options, &)
+          builder = ERRORS.fetch(kind) do
+            raise ArgumentError, "unknown error #{kind.inspect} (one of #{ERRORS.keys.map(&:inspect).join(', ')})"
+          end
+          definition = klass.squishling_definition(method)
+          # Built once now, so a mistyped option fails here and not in the code under test.
+          send(builder, definition, **options)
+          activate(klass, method, -> { raise send(builder, definition, **options) }, &)
+        end
+
+        def clear!
+          LOCK.synchronize { @active.clear }
+        end
+
+        # The stub for a method, called by the module prepended to its class (nil when there is none).
+        def stub_for(klass, method)
+          LOCK.synchronize { @active[[klass, method]] }
+        end
+
+        private
+
+        def activate(klass, method, callable)
+          previous = LOCK.synchronize { @active.delete([klass, method]).tap { @active[[klass, method]] = callable } }
+          hook(klass, method)
+          return klass unless block_given?
+
+          begin
+            yield
+          ensure
+            LOCK.synchronize { previous ? @active[[klass, method]] = previous : @active.delete([klass, method]) }
+          end
+        end
+
+        # Prepended once per method: calls the stub while there is one, and the real method otherwise.
+        def hook(klass, method)
+          LOCK.synchronize { @hooks[[klass, method]] ||= build_hook(klass, method) }
+        end
+
+        # A Wrapper, so Wrapper.implementation looks past it to the real method (its parameter names name the
+        # arguments the model is sent) once the stub is gone.
+        def build_hook(klass, method)
+          hook = Wrapper.new
+          hook.define_method(method) do |*args, **kwargs, &block|
+            stub = Stubs.stub_for(klass, method) if instance_of?(klass) # a subclass keeps its own behavior
+            stub ? stub.call : super(*args, **kwargs, &block)
+          end
+          klass.prepend(hook)
+          hook
+        end
+
+        # The attributes as the method's own typed result, validated by the same path as a Ruby-path return.
+        def typed(definition, attrs, squished:)
+          definition.build_result(attrs, squished:)
+        rescue InvalidOutputError => e
+          raise SetupError, "#{definition.label}: the stubbed result doesn't match its output_schema: " \
+                            "#{e.errors.join('; ')}"
+        rescue ConfigurationError
+          raise SetupError, "#{definition.label} has no output_schema to build a result from"
+        end
+
+        def invalid_output_error(_definition, errors: ["stubbed invalid output"], raw: nil, attempts: 1, models: nil)
+          InvalidOutputError.new(errors: Array(errors), raw:, attempts:, models: models || ([nil] * attempts))
+        end
+
+        def llm_error(definition, message: "stubbed LLM failure")
+          LLMError.new("#{definition.label}: #{message}")
+        end
+
+        def configuration_error(definition, message: "stubbed configuration error")
+          ConfigurationError.new("#{definition.label}: #{message}")
+        end
+
+        def disagreement_error(definition, candidates:, verdict: nil, reason: nil, models: nil)
+          typed = candidates.map { |candidate| typed(definition, candidate, squished: true) }
+          DisagreementError.new(candidates: typed, verdict:, reason:, models:)
+        end
+      end
+    end
+  end
+end