squishling 0.4.0 → 0.5.1

data/lib/squishling/testing/script.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # One scripted line: the requests it answers and what they get, in order. The last answer repeats unless the
+    # rule is #once. Build them through Script (the block Testing.install yields).
+    class Rule
+      # What a rule answers. nil means any. kind is :chat (Adapter#complete) or :judgment (Adapter#judge); role is
+      # nil (an escalation attempt, or any role when the rule names none), :sample, or :judge.
+      Selector = Data.define(:kind, :role, :sample, :model, :provider, :attempt)
+
+      # Rule#take when nothing is left to answer with.
+      EXHAUSTED = Object.new.freeze
+
+      attr_reader :selector
+
+      def initialize(selector)
+        @selector = selector
+        @replies = []
+        @taken = 0
+        @once = false
+      end
+
+      # Answers, in order: a Hash (the model's answer), a String (raw text, malformed or fenced JSON included), nil
+      # (an empty response), a Reply (with a model id and token usage), or an Exception (raised as is).
+      def reply(*replies)
+        @replies.concat(replies)
+        self
+      end
+
+      # A provider failure named by intent (see Failures::INTENTS).
+      def fail(intent)
+        @replies << Failure.new(Failures.check!(intent))
+        self
+      end
+
+      # Running out of answers is an error instead of repeating the last one.
+      def once
+        @once = true
+        self
+      end
+
+      def matches?(kind, call)
+        selector.kind == kind && fits?(selector.role, call[:role]) && fits?(selector.sample, call[:sample]) &&
+          fits?(selector.model, call[:model]) && fits?(selector.provider, call[:provider]) &&
+          fits?(selector.attempt, call[:attempt])
+      end
+
+      # How narrowly the rule aims, lowest first: rules for a sample or the judge, then those naming the most of a
+      # model, a provider, and an attempt, then the rest.
+      def specificity
+        return 0 if selector.sample || selector.role
+
+        4 - [selector.model, selector.provider, selector.attempt].count { |named| !named.nil? }
+      end
+
+      # The next answer, or EXHAUSTED. Called with the script's lock held.
+      def take
+        return EXHAUSTED if @replies.empty? || (@once && @taken >= @replies.size)
+
+        @replies[[@taken, @replies.size - 1].min].tap { @taken += 1 }
+      end
+
+      def empty?
+        @replies.empty?
+      end
+
+      def describe
+        aimed = selector.to_h.compact.except(:kind).map { |key, value| "#{key}: #{value.inspect}" }
+        "#{selector.kind} #{aimed.empty? ? 'any' : aimed.join(', ')}"
+      end
+
+      private
+
+      def fits?(wanted, actual)
+        wanted.nil? || wanted.to_s == actual.to_s
+      end
+    end
+
+    # The judge's rules: a chat judge answers a verdict, a System One judge a judgment with a confidence.
+    class JudgeRules
+      CHOICES = %i[a b neither].freeze
+
+      def initialize(chat, judgment)
+        @chat = chat
+        @judgment = judgment
+      end
+
+      # A chat judge's strict verdict (:a, :b, or :neither) and its reason.
+      def verdict(choice, reason: "scripted verdict")
+        @chat.reply({ "verdict" => check(choice).to_s, "reason" => reason })
+        self
+      end
+
+      # A System One judgment (type: :judgment) choosing :a, :b, or :neither with that probability; the rest of the
+      # probability is split between the other two choices. Pass an Exception to raise it instead.
+      def judgment(choice, confidence: 0.95)
+        unless choice.is_a?(Exception) || confidence.between?(0, 1)
+          raise ArgumentError, "confidence is a probability from 0 to 1, got #{confidence.inspect}"
+        end
+
+        @judgment.reply(choice.is_a?(Exception) ? choice : answer(check(choice), confidence))
+        self
+      end
+
+      # Anything a chat judge's request can answer with (see Rule#reply): off-schema verdicts, raw text, ...
+      def reply(*replies)
+        @chat.reply(*replies)
+        self
+      end
+
+      def fail(intent)
+        @chat.fail(intent)
+        self
+      end
+
+      def fail_judgment(intent)
+        @judgment.fail(intent)
+        self
+      end
+
+      def once
+        @chat.once
+        @judgment.once
+        self
+      end
+
+      private
+
+      def check(choice)
+        return choice.to_sym if CHOICES.include?(choice.to_s.to_sym)
+
+        raise ArgumentError, "a judge choice is one of #{CHOICES.map(&:inspect).join(', ')}, got #{choice.inspect}"
+      end
+
+      def answer(choice, confidence)
+        others = (1 - confidence) / 2.0
+        probabilities = CHOICES.to_h { |key| [key, key == choice ? confidence.to_f : others] }
+        Judgment.new({ winner: Answer.new(choice, probabilities) })
+      end
+    end
+
+    # What a test says the model will do. Rules are matched most specific first (a sample or the judge, then a
+    # named model, then step(:any)), and by declaration order within that.
+    #
+    #   llm.step("flash").reply({ score: 99 }, { score: 5 })   # garbage, then an answer
+    #   llm.step("pro").fail(:rate_limit)
+    #   llm.sample("a").reply(valid_a)
+    #   llm.judge.verdict(:a)
+    class Script
+      def initialize
+        @rules = []
+      end
+
+      # Escalation attempts (and any other chat request no sample or judge rule claims) on a model, a provider, or
+      # :any model; attempt: narrows it to one attempt of the escalation (counting from 1).
+      def step(model = :any, provider: nil, attempt: nil)
+        add(:chat, model: model == :any ? nil : model, provider:, attempt:)
+      end
+
+      # A harness sample's requests ("a", "b", ...). Each sample has its own queue, so the order the sample threads
+      # happen to run in can't change what a sample gets.
+      def sample(name)
+        add(:chat, role: :sample, sample: name.to_s)
+      end
+
+      # The judge of :judged_squishsum, chat or System One.
+      def judge
+        @judge ||= JudgeRules.new(add(:chat, role: :judge), add(:judgment, role: :judge))
+      end
+
+      # The rule that answers a request of this kind, or nil. Called with the lock held (see Adapter).
+      def rule_for(kind, call)
+        candidates = @rules.each_with_index.select { |rule, _| rule.matches?(kind, call) }
+        candidates.min_by { |rule, index| [rule.specificity, index] }&.first
+      end
+
+      def describe
+        @rules.empty? ? "nothing" : @rules.map(&:describe).join("; ")
+      end
+
+      private
+
+      def add(kind, role: nil, sample: nil, model: nil, provider: nil, attempt: nil)
+        Rule.new(Rule::Selector.new(kind:, role:, sample:, model:, provider:, attempt:)).tap { |rule| @rules << rule }
+      end
+    end
+  end
+end

data/lib/squishling/testing/state.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # What the helpers changed in Squishling.config (the adapter and observer), remembered so Testing.reset! puts
+    # back exactly that: not an app's other settings (which Squishling.reset_config! would also clear), and not a
+    # value a test assigned itself after the helpers did.
+    module State
+      LOCK = Mutex.new
+      private_constant :LOCK
+
+      @entries = {} # setting => [what it was before the helpers, what the helpers set it to]
+      @config = nil
+
+      class << self
+        # Sets a Squishling.config setting, remembering what it was before the first change.
+        def set(setting, value)
+          LOCK.synchronize do
+            config = Squishling.config
+            @entries = {} unless config.equal?(@config) # a replaced config is already pristine
+            @config = config
+            original = original_of(config, setting)
+            config.public_send(:"#{setting}=", value)
+            @entries[setting] = [original, value]
+          end
+        end
+
+        # Puts back the settings still holding what the helpers set; leaves a config that has been replaced alone.
+        def restore!
+          config, entries = LOCK.synchronize do
+            [@config, @entries].tap do
+              @entries = {}
+              @config = nil
+            end
+          end
+          return unless config.equal?(Squishling.config)
+
+          entries.each do |setting, (original, installed)|
+            config.public_send(:"#{setting}=", original) if config.public_send(setting).equal?(installed)
+          end
+        end
+
+        private
+
+        # What to put back: the value before the helpers' first change, unless something else was assigned since.
+        def original_of(config, setting)
+          current = config.public_send(setting)
+          entry = @entries[setting]
+          entry && current.equal?(entry.last) ? entry.first : current
+        end
+      end
+    end
+
+    # What a test asked squawk to record: every attempt's raw output, error, and metadata (role, sample, attempt,
+    # final, usage, ...), and the report of a call stopped by a token budget or a disagreement (stopped).
+    # Use Testing.with_squawk, which puts the previous hook back.
+    class SquawkCapture
+      Call = Data.define(:output, :error, :metadata)
+
+      def initialize
+        @lock = Mutex.new
+        @calls = []
+      end
+
+      def call(output: nil, error: nil, metadata: nil)
+        @lock.synchronize { @calls << Call.new(output:, error:, metadata:) }
+      end
+
+      def calls
+        @lock.synchronize { @calls.dup }
+      end
+    end
+  end
+end

data/lib/squishling/testing/stubs.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # Replaces a squished method for the tests of its callers: the method returns (or raises) what the test says and
+    # never reaches Squishling, but what it returns is a real typed result, validated against the method's output
+    # schema, so the stand-in can't drift from the real result type. Cleared by Testing.reset!.
+    module Stubs
+      LOCK = Mutex.new
+      private_constant :LOCK
+
+      # [class, method name] => callable producing the result, or raising the error
+      @active = {}
+      # [class, method name] => the module prepended to the class. A prepended module can't be removed, so it stays
+      # (inert once its stub is cleared) and keeps its class alive: stub the classes of a suite, not one per test.
+      @hooks = {}
+
+      ERRORS = {
+        invalid_output: :invalid_output_error, llm_error: :llm_error, configuration: :configuration_error,
+        disagreement: :disagreement_error
+      }.freeze
+
+      class << self
+        # A result of the method's schema, with these attributes (squished: false for one the Ruby path returned).
+        def result(klass, attrs, method: :call, squished: true, &)
+          definition = klass.squishling_definition(method)
+          built = typed(definition, attrs, squished:)
+          activate(klass, method, -> { built }, &)
+        end
+
+        # The method raises this kind of error: :invalid_output (errors:, raw:, attempts:, models:), :llm_error
+        # (message:), :configuration (message:), or :disagreement (candidates:, verdict:, reason:, models:).
+        def error(klass, kind, method: :call, **options, &)
+          builder = ERRORS.fetch(kind) do
+            raise ArgumentError, "unknown error #{kind.inspect} (one of #{ERRORS.keys.map(&:inspect).join(', ')})"
+          end
+          definition = klass.squishling_definition(method)
+          # Built once now, so a mistyped option fails here and not in the code under test.
+          send(builder, definition, **options)
+          activate(klass, method, -> { raise send(builder, definition, **options) }, &)
+        end
+
+        def clear!
+          LOCK.synchronize { @active.clear }
+        end
+
+        # The stub for a method, called by the module prepended to its class (nil when there is none).
+        def stub_for(klass, method)
+          LOCK.synchronize { @active[[klass, method]] }
+        end
+
+        private
+
+        def activate(klass, method, callable)
+          previous = LOCK.synchronize { @active.delete([klass, method]).tap { @active[[klass, method]] = callable } }
+          hook(klass, method)
+          return klass unless block_given?
+
+          begin
+            yield
+          ensure
+            LOCK.synchronize { previous ? @active[[klass, method]] = previous : @active.delete([klass, method]) }
+          end
+        end
+
+        # Prepended once per method: calls the stub while there is one, and the real method otherwise.
+        def hook(klass, method)
+          LOCK.synchronize { @hooks[[klass, method]] ||= build_hook(klass, method) }
+        end
+
+        # A Wrapper, so Wrapper.implementation looks past it to the real method (its parameter names name the
+        # arguments the model is sent) once the stub is gone.
+        def build_hook(klass, method)
+          hook = Wrapper.new
+          hook.define_method(method) do |*args, **kwargs, &block|
+            stub = Stubs.stub_for(klass, method) if instance_of?(klass) # a subclass keeps its own behavior
+            stub ? stub.call : super(*args, **kwargs, &block)
+          end
+          klass.prepend(hook)
+          hook
+        end
+
+        # The attributes as the method's own typed result, validated by the same path as a Ruby-path return.
+        def typed(definition, attrs, squished:)
+          definition.build_result(attrs, squished:)
+        rescue InvalidOutputError => e
+          raise SetupError, "#{definition.label}: the stubbed result doesn't match its output_schema: " \
+                            "#{e.errors.join('; ')}"
+        rescue ConfigurationError
+          raise SetupError, "#{definition.label} has no output_schema to build a result from"
+        end
+
+        def invalid_output_error(_definition, errors: ["stubbed invalid output"], raw: nil, attempts: 1, models: nil)
+          InvalidOutputError.new(errors: Array(errors), raw:, attempts:, models: models || ([nil] * attempts))
+        end
+
+        def llm_error(definition, message: "stubbed LLM failure")
+          LLMError.new("#{definition.label}: #{message}")
+        end
+
+        def configuration_error(definition, message: "stubbed configuration error")
+          ConfigurationError.new("#{definition.label}: #{message}")
+        end
+
+        def disagreement_error(definition, candidates:, verdict: nil, reason: nil, models: nil)
+          typed = candidates.map { |candidate| typed(definition, candidate, squished: true) }
+          DisagreementError.new(candidates: typed, verdict:, reason:, models:)
+        end
+      end
+    end
+  end
+end

data/lib/squishling/testing.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "squishling"
+
+require_relative "testing/errors"
+require_relative "testing/reply"
+require_relative "testing/script"
+require_relative "testing/request"
+require_relative "testing/observer"
+require_relative "testing/adapter"
+require_relative "testing/state"
+require_relative "testing/checks"
+require_relative "testing/stubs"
+require_relative "testing/direct"
+require_relative "testing/fixtures"
+require_relative "testing/helpers"
+require_relative "testing/minitest"
+require_relative "testing/rspec"
+
+module Squishling
+  # Test support, opt-in: `require "squishling/testing"` (it is not loaded with `require "squishling"`, and needs no
+  # test framework). A scripted adapter answers in place of a provider and records what was sent to it, plus stubs
+  # for the callers of squished methods, schema-driven fixtures, and assertions. See docs/testing.md and the
+  # Squishling::Testing::Minitest and ::RSpec mixins.
+  module Testing
+    class << self
+      # Makes the scripted Adapter the one every squished call goes through, and returns it. The block scripts it
+      # (see Script); #script adds to it later.
+      def install(adapter = Adapter.new)
+        yield adapter.script if block_given?
+        State.set(:observer, Observer.new(chained_observer(Squishling.config.observer)))
+        State.set(:adapter, adapter)
+        adapter
+      end
+
+      # Any request that reaches the provider seam raises ProviderCallError, naming the squished method. Call it in a
+      # suite-wide before hook: a test that forgot to script its calls then fails at the call, not far from it.
+      def forbid_provider_calls!
+        State.set(:adapter, ForbiddenAdapter.new)
+      end
+
+      # Runs the block with provider calls forbidden and returns the calls (the squished methods' labels) that tried
+      # to reach the provider, even if the block rescued the error. The adapter is put back afterwards.
+      def no_llm_call
+        config = Squishling.config
+        previous = config.adapter
+        guard = ForbiddenAdapter.new
+        config.adapter = guard
+        begin
+          yield
+        rescue ProviderCallError
+          nil
+        ensure
+          config.adapter = previous
+        end
+        guard.attempts
+      end
+
+      # See Stubs.result: Klass.call returns a real, schema-validated result of the method's type.
+      def stub_result(klass, attrs = nil, method: :call, squished: true, **more, &)
+        Stubs.result(klass, Schema.jsonify((attrs || {}).merge(more)), method:, squished:, &)
+      end
+
+      # See Stubs.error: Klass.call raises the real error class with realistic attributes.
+      def stub_error(klass, kind, **, &)
+        Stubs.error(klass, kind, **, &)
+      end
+
+      # The messages squish_validate and the schema reject this reply with. See Direct.
+      def validate(klass, reply, **inputs)
+        Direct.validate(klass, reply, **inputs)
+      end
+
+      def fallback_for(klass, error, **inputs)
+        Direct.fallback_for(klass, error, **inputs)
+      end
+
+      def preview(klass, **inputs)
+        Direct.preview(klass, **inputs)
+      end
+
+      def valid_reply_for(source, **overrides)
+        Fixtures.valid_reply_for(source, **overrides)
+      end
+
+      def invalid_reply_for(source, **)
+        Fixtures.invalid_reply_for(source, **)
+      end
+
+      # A recorder to assign as config.squawk (or give to with_squawk): every attempt's output, error and metadata.
+      def capture_squawk
+        SquawkCapture.new
+      end
+
+      # Runs the block with the capture as config.squawk, and puts the previous hook back. A squawk set on a class or
+      # a method still takes precedence over the configured one.
+      def with_squawk(capture = capture_squawk)
+        config = Squishling.config
+        previous = config.squawk
+        config.squawk = capture
+        yield capture
+        capture
+      ensure
+        config.squawk = previous
+      end
+
+      # Puts back what the helpers changed (the adapter and observer) and clears the stubs. The mixins do this
+      # after every test; call it yourself with a custom setup.
+      def reset!
+        Stubs.clear!
+        State.restore!
+      end
+
+      private
+
+      # An observer the app configured itself (an object, or :coolhand) is kept in the chain; the default Coolhand
+      # auto-detection, and our own observer from an earlier install, are not.
+      def chained_observer(observer)
+        return observer.previous if observer.is_a?(Observer)
+        return Squishling::Observer.resolve(observer) if observer.is_a?(Symbol)
+
+        observer if observer.respond_to?(:around)
+      end
+    end
+  end
+end

data/lib/squishling/verification.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Squishling
+  # What Squishling.verify found: the problems in the squished classes' setup, none when it is sound.
+  module Verification
+    # One thing wrong. label names where ("Triage#call", or "adapter" for a setup that isn't one class's).
+    Problem = Data.define(:label, :message) do
+      # The existing messages already start with the label; say it once.
+      def to_s
+        message.start_with?(label) ? message : "#{label}: #{message}"
+      end
+    end
+
+    class Report
+      attr_reader :problems
+
+      def initialize(problems)
+        @problems = problems.dup.freeze
+        freeze
+      end
+
+      def ok?
+        problems.empty?
+      end
+
+      def size
+        problems.size
+      end
+
+      # A lint-style listing, one problem per line.
+      def to_s
+        return "Squishling: no problems found" if ok?
+
+        lines = problems.map { |problem| "  #{problem}" }
+        "Squishling found #{size} #{size == 1 ? 'problem' : 'problems'}:\n#{lines.join("\n")}"
+      end
+    end
+  end
+end