squishling 0.4.0 → 0.5.1

data/lib/squishling/testing/checks.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # The assertions, independent of any test framework: each returns a failure message, or nil when it holds. The
+    # Minitest and RSpec mixins turn a message into a failure in their own way.
+    module Checks
+      module_function
+
+      # The requests of a fake adapter (or an Array of them) sent exactly these arguments, as JSON would carry them.
+      def sent_arguments(source, expected, request: 0)
+        sent = requests_of(source)[request]
+        return "expected request ##{request} to have been sent, but only #{requests_of(source).size} were" unless sent
+
+        wanted = Schema.jsonify(expected)
+        return nil if sent.arguments == wanted
+
+        "expected the model to be sent arguments #{wanted.inspect}, but it was sent #{sent.arguments.inspect}"
+      end
+
+      def call_count(source, expected)
+        actual = requests_of(source).size
+        actual == expected ? nil : "expected #{expected} LLM call(s), but there were #{actual}"
+      end
+
+      # Runs the block with provider calls forbidden: a failure message when it tried to make one.
+      def no_llm_call(&)
+        attempts = Testing.no_llm_call(&)
+        return nil if attempts.empty?
+
+        "expected no LLM call, but #{attempts.uniq.join(', ')} reached the provider (#{attempts.size} request(s))"
+      end
+
+      def requests_of(source)
+        source.respond_to?(:requests) ? source.requests : Array(source)
+      end
+    end
+  end
+end

data/lib/squishling/testing/direct.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # Runs the parts of a squished method that don't need a model, for unit tests: the output checks, the fallback,
+    # and the prompt. Each takes the method's arguments as args: and keywords (as the method is called), and a
+    # receiver: for a class whose constructor takes arguments.
+    module Direct
+      class << self
+        # The messages squish_validate and the output schema reject this reply with ([] when it would be accepted).
+        def validate(klass, reply, method: :call, receiver: nil, args: [], **kwargs)
+          definition = klass.squishling_definition(method)
+          schema = definition.schema or raise SetupError, "#{definition.label} has no output_schema"
+          inputs = definition.bind_arguments(args, kwargs)
+          prompt = Invoker::Prompt.new(instructions: nil, schema:, input: nil, role: nil)
+          OutputCheck.new(definition, receiver || klass.new, inputs).call(reply, prompt).last.map(&:message)
+        end
+
+        # What squish_fallback returns for this error, typed like any result of the method.
+        def fallback_for(klass, error, method: :call, receiver: nil, args: [], **kwargs)
+          definition = klass.squishling_definition(method)
+          handler = definition.fallback or raise SetupError, "#{definition.label} has no squish_fallback"
+          inputs = definition.bind_arguments(args, kwargs)
+          definition.coerce((receiver || klass.new).instance_exec(error, **inputs, &handler))
+        end
+
+        # The first request a call would make, without making it: { system_prompt:, user_message:, output_schema:,
+        # step: } (the step is the ModelPath::Step with the model, provider and params; rotation isn't advanced).
+        def preview(klass, method: :call, receiver: nil, args: [], **kwargs)
+          definition = klass.squishling_definition(method)
+          receiver ||= klass.new
+          inputs = definition.bind_arguments(args, kwargs)
+          prompt = Invoker.new(definition, receiver, inputs).prompt
+          { system_prompt: prompt.instructions, user_message: prompt.input,
+            output_schema: prompt.schema.llm_schema["schema"],
+            step: definition.escalation_path(receiver, inputs, advance: false).first }
+        end
+      end
+    end
+  end
+end

data/lib/squishling/testing/errors.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # What the test helpers raise. None of these is a Squishling::Error, so squish_fallback, which absorbs
+    # InvalidOutputError and LLMError, can never hide a test setup mistake.
+    class Error < StandardError; end
+
+    # A request reached the test adapter and no script covers it.
+    class UnscriptedRequestError < Error; end
+
+    # A request reached the provider seam while provider calls were forbidden (see Testing.forbid_provider_calls!).
+    class ProviderCallError < Error; end
+
+    # A stub, a fixture, or a direct check was asked for something the method's schema doesn't allow.
+    class SetupError < Error; end
+  end
+end

data/lib/squishling/testing/fixtures.rb

@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # Replies derived from an output schema, so a test that doesn't care about the content stays valid when the schema
+    # changes, and a test of "out of range gets re-asked" doesn't hand-write a bad payload. Replies are Hashes with
+    # string keys, as JSON would carry them, and are always checked against the schema before they are returned.
+    module Fixtures
+      FORMATS = {
+        "date-time" => "2024-01-01T00:00:00Z", "date" => "2024-01-01", "time" => "00:00:00Z",
+        "email" => "user@example.com", "uri" => "https://example.com", "uuid" => "00000000-0000-4000-8000-000000000000"
+      }.freeze
+      MAX_DEPTH = 8
+      WRONG_VALUES = ["not valid", -1_000_000_007, [], {}, true, nil].freeze
+
+      class << self
+        # A reply that satisfies the schema of a squished class's method, a Squishling::Schema, or a schema Hash.
+        # overrides replace top-level attributes.
+        def valid_reply_for(source, method: :call, **overrides)
+          schema = schema_of(source, method)
+          reply = Generator.new(schema.json_schema).value
+          reply = reply.merge(overrides.transform_keys(&:to_s)) if reply.is_a?(Hash)
+          verify!(schema, reply, "valid")
+        end
+
+        # A valid reply with one top-level attribute (violating:) broken: out of its enum or range, the wrong type,
+        # null, or missing. how: (:enum, :range, :type, :null, :missing) picks the break; by default the first that
+        # the schema rejects. Overrides are applied to the valid reply first.
+        def invalid_reply_for(source, violating:, how: nil, method: :call, **overrides)
+          schema = schema_of(source, method)
+          base = valid_reply_for(schema, **overrides)
+          key = violating.to_s
+          raise SetupError, "the schema has no property #{key.inspect} to violate" unless schema.json_schema.fetch(
+            "properties", {}
+          ).key?(key)
+
+          candidates = breaks(schema.json_schema, key, how)
+          broken = candidates.lazy.map { |value| mutate(base, key, value) }.find { |reply| schema.validate(reply).any? }
+          broken or raise SetupError,
+            "couldn't find a value of #{key.inspect} that the schema rejects#{" (how: #{how.inspect})" if how}"
+        end
+
+        private
+
+        def schema_of(source, method)
+          case source
+          when Schema then source
+          when Class then source.squishling_definition(method).schema or raise SetupError,
+            "#{source} has no output_schema"
+          else Schema.for(source)
+          end
+        end
+
+        def verify!(schema, reply, kind)
+          errors = schema.validate(reply)
+          return reply if errors.empty?
+
+          raise SetupError,
+            "couldn't build a #{kind} reply (#{errors.join('; ')}); pass the attributes it needs as overrides"
+        end
+
+        def mutate(base, key, value)
+          value.equal?(:missing) ? base.except(key) : base.merge(key => value)
+        end
+
+        def breaks(json_schema, key, how)
+          node = Generator.new(json_schema).node_for(json_schema.fetch("properties").fetch(key))
+          all = { enum: enum_breaks(node), range: range_breaks(node), type: WRONG_VALUES.compact, null: [nil],
+                  missing: [:missing] }
+          return all.fetch(how) { raise ArgumentError, "how: is one of #{all.keys.map(&:inspect).join(', ')}" } if how
+
+          all.values.flatten(1)
+        end
+
+        def enum_breaks(node)
+          values = node["enum"] ? ["not in the enum"] : []
+          node.key?("const") ? values + ["not the const"] : values
+        end
+
+        def range_breaks(node)
+          bounds = node.slice("minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum", "minLength", "maxLength",
+            "minItems", "maxItems")
+          bounds.flat_map { |name, limit| beyond(name, limit) }
+        end
+
+        def beyond(name, limit)
+          case name
+          when "minimum", "exclusiveMinimum" then [limit - 1]
+          when "maximum", "exclusiveMaximum" then [limit + 1]
+          when "minLength" then ["x" * [limit - 1, 0].max]
+          when "maxLength" then ["x" * (limit + 1)]
+          when "minItems" then [[nil] * [limit - 1, 0].max]
+          when "maxItems" then [[nil] * (limit + 1)]
+          end
+        end
+      end
+
+      # Builds one value that satisfies a JSON schema (see Fixtures).
+      class Generator
+        def initialize(root)
+          @root = root
+          @expanding = []
+        end
+
+        # The value for a node. A $ref being expanded is remembered, so a recursive schema (a node whose child is
+        # the same node or null) ends at the null instead of nesting forever.
+        def value(node = @root, depth = 0)
+          ref = node["$ref"]
+          @expanding.push(ref) if ref
+          generate(node_for(node), depth)
+        ensure
+          @expanding.pop if ref
+        end
+
+        # The schema node with a root-relative $ref followed.
+        def node_for(node)
+          ref = node["$ref"]
+          return node unless ref
+
+          path = ref.delete_prefix("#").split("/").reject(&:empty?)
+          node.except("$ref").merge(path.reduce(@root) do |memo, part|
+            memo.fetch(part.gsub("~1", "/").gsub("~0", "~"))
+          end)
+        end
+
+        private
+
+        def generate(node, depth)
+          return node["const"] if node.key?("const")
+          return node["enum"].first if node["enum"]
+          return from_branch(node, depth) if node["anyOf"] || node["oneOf"]
+          return from_all_of(node, depth) if node["allOf"]
+
+          typed(node, depth)
+        end
+
+        def from_branch(node, depth)
+          branches = node["anyOf"] || node["oneOf"]
+          open = branches.reject { |branch| branch["$ref"] && @expanding.include?(branch["$ref"]) }
+          chosen = open.find { |branch| branch["type"] != "null" } || open.first || branches.first
+          value(chosen, depth + 1)
+        end
+
+        def from_all_of(node, depth)
+          parts = node["allOf"].map { |part| value(part, depth + 1) }.grep(Hash)
+          parts.reduce({}) { |memo, part| memo.merge(part) }
+        end
+
+        def typed(node, depth)
+          case primary_type(node)
+          when "object" then object(node, depth)
+          when "array" then array(node, depth)
+          when "string" then string(node)
+          when "integer", "number" then number(node)
+          when "boolean" then true
+          end
+        end
+
+        def primary_type(node)
+          types = Array(node["type"])
+          types.find { |type| type != "null" } || (types.first if types.any?) || inferred_type(node)
+        end
+
+        def inferred_type(node)
+          return "object" if node["properties"]
+
+          "array" if node["items"]
+        end
+
+        def object(node, depth)
+          return {} if depth > MAX_DEPTH
+
+          (node["properties"] || {}).transform_values { |property| value(property, depth + 1) }
+        end
+
+        def array(node, depth)
+          return [] if depth > MAX_DEPTH || !node["items"]
+
+          Array.new(node["minItems"].to_i) { value(node["items"], depth + 1) }
+        end
+
+        def string(node)
+          text = FORMATS.fetch(node["format"], "example")
+          text = text.ljust(node["minLength"], "x") if node["minLength"]
+          node["maxLength"] ? text[0, node["maxLength"]] : text
+        end
+
+        def number(node)
+          number = clamp_up(0, node["minimum"], node["exclusiveMinimum"])
+          number = clamp_down(number, node["maximum"], node["exclusiveMaximum"])
+          primary_type(node) == "integer" ? number.ceil : number
+        end
+
+        def clamp_up(number, minimum, exclusive)
+          number = minimum if minimum && number < minimum
+          exclusive && number <= exclusive ? exclusive + 1 : number
+        end
+
+        def clamp_down(number, maximum, exclusive)
+          number = maximum if maximum && number > maximum
+          exclusive && number >= exclusive ? exclusive - 1 : number
+        end
+      end
+    end
+  end
+end

data/lib/squishling/testing/helpers.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # The test-facing helpers, shared by the Minitest and RSpec mixins. A mixin supplies #squishling_failure(message),
+    # which fails the current test.
+    module Helpers
+      # Installs a scripted adapter and returns it. The block scripts it (see Script); the adapter's #requests,
+      # #conversations and #arguments show what was sent.
+      def fake_squishling(&)
+        Testing.install(&)
+      end
+
+      # Any LLM request raises until a test installs a fake, naming the squished method that made it.
+      def forbid_provider_calls!
+        Testing.forbid_provider_calls!
+      end
+
+      def assert_sent_arguments(source, expected, request: 0)
+        failure = Checks.sent_arguments(source, expected, request:)
+        squishling_failure(failure) if failure
+      end
+
+      def assert_llm_calls(source, expected)
+        failure = Checks.call_count(source, expected)
+        squishling_failure(failure) if failure
+      end
+
+      def assert_no_llm_call(&)
+        failure = Checks.no_llm_call(&)
+        squishling_failure(failure) if failure
+      end
+
+      def stub_squishling_result(klass, attrs = nil, **, &)
+        Testing.stub_result(klass, attrs, **, &)
+      end
+
+      def stub_squishling_error(klass, kind, **, &)
+        Testing.stub_error(klass, kind, **, &)
+      end
+
+      def squishling_validate(klass, reply, **inputs)
+        Testing.validate(klass, reply, **inputs)
+      end
+
+      def squishling_fallback_for(klass, error, **inputs)
+        Testing.fallback_for(klass, error, **inputs)
+      end
+
+      def squishling_preview(klass, **inputs)
+        Testing.preview(klass, **inputs)
+      end
+
+      def valid_reply_for(source, **overrides)
+        Testing.valid_reply_for(source, **overrides)
+      end
+
+      def invalid_reply_for(source, **)
+        Testing.invalid_reply_for(source, **)
+      end
+
+      def capture_squawk
+        Testing.capture_squawk
+      end
+
+      def with_squawk(capture = capture_squawk, &)
+        Testing.with_squawk(capture, &)
+      end
+    end
+  end
+end

data/lib/squishling/testing/minitest.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # The helpers for Minitest: include it in a test case (or ActiveSupport::TestCase). What it installs (the
+    # adapter, observer, and stubs) is put back after each test.
+    #
+    #   class RegexInferrerTest < Minitest::Test
+    #     include Squishling::Testing::Minitest
+    #
+    #     def test_retries_a_bad_regex
+    #       fake = fake_squishling { |llm| llm.step(:any).reply({ regex: "[bad(" }, { regex: "Run.*" }) }
+    #       assert_equal "Run.*", RegexInferrer.call(sample_turns: turns).regex
+    #       assert_sent_arguments fake, { sample_turns: turns }
+    #     end
+    #   end
+    module Minitest
+      include Helpers
+
+      def after_teardown
+        Testing.reset!
+        super
+      end
+
+      private
+
+      def squishling_failure(message)
+        flunk message
+      end
+    end
+  end
+end

data/lib/squishling/testing/observer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # Tells the test adapter which request it is answering. An Adapter::Request carries no role, sample, or attempt,
+    # but the Observer::Call that wraps it does, on the thread that makes the request: this observer keeps it where
+    # Adapter#complete can read it, and passes the request on to any observer the app configured.
+    class Observer
+      KEY = :squishling_testing_call
+
+      # The call being made on this thread, or nil outside a request.
+      def self.current
+        Thread.current[KEY]
+      end
+
+      attr_reader :previous
+
+      def initialize(previous = nil)
+        @previous = previous
+      end
+
+      def around(call, &)
+        outer = Thread.current[KEY]
+        Thread.current[KEY] = call
+        @previous ? @previous.around(call, &) : yield
+      ensure
+        Thread.current[KEY] = outer
+      end
+    end
+  end
+end

data/lib/squishling/testing/reply.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # A scripted answer with more than content: the model id the provider reports and its token counts, so the
+    # squawk and observer usage paths are covered. usage is a Hash such as { input_tokens: 12, output_tokens: 30 }.
+    class Reply
+      attr_reader :content, :model, :usage
+
+      def initialize(content = nil, model: nil, usage: nil)
+        @content = content
+        @model = model
+        @usage = usage
+      end
+    end
+
+    # A scripted provider failure, named by what went wrong (see Failures).
+    Failure = Data.define(:intent)
+
+    # The answer to the "winner" question of a System One judgment.
+    Answer = Struct.new(:choice, :probabilities)
+
+    # What Adapter#judge returns: answers by question name, like RubyLLM's Judgment.
+    Judgment = Struct.new(:answers) do
+      def [](name)
+        answers[name]
+      end
+    end
+
+    # The provider failures a script can name by intent rather than by the error class of any one SDK. Each becomes
+    # the Squishling error an adapter raises for it: LLMError for what is worth another attempt, ConfigurationError
+    # for what is not. Messages carry the failure and its status, like the real adapters' do.
+    module Failures
+      # intent => [error class, what happened, HTTP status or nil]
+      INTENTS = {
+        rate_limit: [LLMError, "rate limited", 429],
+        server_error: [LLMError, "server error", 500],
+        overloaded: [LLMError, "provider overloaded", 529],
+        timeout: [LLMError, "request timed out", nil],
+        context_length: [LLMError, "context length exceeded", 400],
+        bad_request: [ConfigurationError, "bad request", 400],
+        unauthorized: [ConfigurationError, "unauthorized", 401],
+        forbidden: [ConfigurationError, "forbidden", 403]
+      }.freeze
+
+      module_function
+
+      def check!(intent)
+        return intent if INTENTS.key?(intent)
+
+        raise ArgumentError, "unknown failure #{intent.inspect} (one of #{INTENTS.keys.map(&:inspect).join(', ')})"
+      end
+
+      def build(intent, label)
+        klass, text, status = INTENTS.fetch(intent)
+        klass.new("#{label}: #{text}#{" (HTTP #{status})" if status}")
+      end
+    end
+  end
+end

data/lib/squishling/testing/request.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # Run's follow-up messages (a private constant of Run): the helpers read what was sent through their parsers.
+    MESSAGES = Run.const_get(:Messages)
+    private_constant :MESSAGES
+
+    # One provider request as the test adapter saw it: what left the process, and what came back.
+    #
+    # arguments and context are the method's arguments and the squish_context / squish! context values the model was
+    # sent. For a judge they are those of the call being judged, and candidates holds the two samples. messages is
+    # the conversation as sent (empty for a System One judgment, whose payload is the input). conversation is the
+    # index of the chat in Adapter#conversations (nil for a judgment): a retry is the same conversation, an
+    # escalation a new one. reply is the Adapter::Response, or the error raised instead.
+    Request = Data.define(:label, :type, :role, :sample, :attempt, :attempts, :model, :provider, :params,
+      :system_prompt, :output_schema, :messages, :conversation, :payload, :reply) do
+      def arguments
+        input["arguments"]
+      end
+
+      def context
+        input["context"]
+      end
+
+      def candidates
+        payload["candidates"]
+      end
+
+      # The validation messages this request carried back to the model: a retry in the same conversation, or an
+      # escalation to the next step. Empty for a first attempt.
+      def errors
+        (retry_errors || escalation&.errors) || []
+      end
+
+      # The rejected output forwarded from the previous step (text, capped), or nil when none was.
+      def forwarded_rejected
+        escalation&.rejected
+      end
+
+      def failed?
+        reply.is_a?(Exception)
+      end
+
+      def judge?
+        role == :judge
+      end
+
+      # The JSON payload of the first user message, parsed (an escalation message wraps it in its own text).
+      def self.payload_of(messages)
+        text = messages.first.fetch(:content)
+        JSON.parse(MESSAGES.parse_escalation(text)&.input || text)
+      rescue JSON::ParserError
+        {}
+      end
+
+      private
+
+      # The method's arguments and context: the payload itself, or inside a judge's "input".
+      def input
+        judge? ? payload.fetch("input", {}) : payload
+      end
+
+      def retry_errors
+        text = messages.last&.fetch(:content, nil)
+        messages.size > 1 && text ? MESSAGES.parse_retry(text) : nil
+      end
+
+      def escalation
+        text = messages.first&.fetch(:content, nil)
+        text && MESSAGES.parse_escalation(text)
+      end
+    end
+  end
+end

data/lib/squishling/testing/rspec.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Testing
+    # The helpers for RSpec (no rspec-mocks needed): `config.include Squishling::Testing::RSpec`, or include it in a
+    # group. What it installs (the adapter, observer, and stubs) is put back after each example.
+    #
+    #   RSpec.describe RegexInferrer do
+    #     it "retries a bad regex" do
+    #       fake = fake_squishling { |llm| llm.step(:any).reply({ regex: "[bad(" }, { regex: "Run.*" }) }
+    #       expect(described_class.call(sample_turns: turns).regex).to eq("Run.*")
+    #       assert_sent_arguments fake, { sample_turns: turns }
+    #     end
+    #   end
+    module RSpec
+      include Helpers
+
+      def self.included(group)
+        super
+        group.after { Testing.reset! }
+      end
+
+      private
+
+      def squishling_failure(message)
+        raise ::RSpec::Expectations::ExpectationNotMetError, message
+      end
+    end
+  end
+end