squishling 0.3.0 → 0.5.0

data/lib/squishling/observers/coolhand.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Squishling
+  module Observers
+    # Declares each request's workload and template to Coolhand (Coolhand.with_context), so Coolhand files the
+    # logged call under them instead of matching it to a template. The workload is the squished method
+    # ("Class#method", described as that, plus "(rescue)" when squish! handed the call over); the template is the
+    # call's place in it: "Class#method/squish:L2" (an escalation attempt), "/sample:a", or "/judge". The model
+    # isn't in the key: it's on the log, and the place is the stable identity.
+    # Used automatically when the coolhand gem is loaded and configured (see Observer.resolve). See docs/coolhand.md.
+    class Coolhand
+      REPORTER = "squishling"
+
+      # Coolhand rejects keys longer than this.
+      MAX_KEY_LENGTH = 255
+
+      # Whether Coolhand is loaded, new enough to declare workloads, and set up to send logs.
+      def self.available?
+        return false unless defined?(::Coolhand) && ::Coolhand.respond_to?(:with_context)
+
+        configuration = ::Coolhand.respond_to?(:configuration) && ::Coolhand.configuration
+        return false unless configuration
+
+        configuration.enabled != false && !configuration.api_key.to_s.strip.empty?
+      rescue StandardError
+        false # nobody asked for this observer, so a Coolhand that can't say whether it's set up isn't ours to break
+      end
+
+      def self.load_library
+        require "coolhand"
+      end
+      private_class_method :load_library
+
+      # Raises ConfigurationError when the coolhand gem is missing or too old to declare workloads.
+      def initialize
+        self.class.send(:load_library) unless defined?(::Coolhand)
+        return if ::Coolhand.respond_to?(:with_context)
+
+        raise ConfigurationError, "observer :coolhand needs a coolhand gem version with Coolhand.with_context " \
+                                  "(declared workloads); upgrade the coolhand gem"
+      rescue LoadError
+        raise ConfigurationError, "observer :coolhand needs the coolhand gem; add `gem \"coolhand\"` to your Gemfile"
+      end
+
+      def around(call, &)
+        workload = call.label
+        template = "#{workload}/#{role_key(call)}"
+        return yield unless declarable?(workload) && declarable?(template)
+
+        ::Coolhand.with_context(
+          reporter: REPORTER,
+          workload: { key: workload, name: workload, description: description(call) },
+          template: { key: template, name: "#{workload} · #{role_name(call)}" },
+          &
+        )
+      end
+
+      private
+
+      # The squished method, marked "(rescue)" when squish! handed it to the LLM. Not the purpose text, which can be
+      # a proc that differs per call.
+      def description(call)
+        call.handed_off ? "#{call.label} (rescue)" : call.label
+      end
+
+      # Coolhand rejects, and drops the whole log for, a key over the limit, with surrounding whitespace, or with
+      # a leading, trailing or doubled "/".
+      def declarable?(key)
+        key.length <= MAX_KEY_LENGTH && key == key.strip && !key.start_with?("/") && !key.end_with?("/") &&
+          !key.include?("//")
+      end
+
+      def role_key(call)
+        case call.role
+        when :sample then "sample:#{call.sample}"
+        when :judge then "judge"
+        else "squish:L#{call.attempt}"
+        end
+      end
+
+      def role_name(call)
+        role_key(call).tr(":", " ")
+      end
+    end
+  end
+end

data/lib/squishling/output_check.rb

@@ -5,6 +5,7 @@ module Squishling
   # method's squish_validate check.
   class OutputCheck
     MAX_ERRORS = 20
+    EMPTY = "response was empty"
     UNREPRESENTABLE = "response contained a value JSON can't represent (such as Infinity or invalid UTF-8)"
 
     def initialize(definition, receiver, inputs)
@@ -13,32 +14,34 @@ module Squishling
       @inputs = inputs
     end
 
-    # Parses and validates a response: [typed result, []] when it passes, [nil or result, errors] otherwise.
+    # Parses and validates a response: [typed result, []] when it passes, [nil or result, error Details] otherwise.
     # The judge's role skips squish_validate, which checks the operation's output, not a verdict.
     def call(content, prompt)
       data, errors = parse(content)
-      errors = prompt.schema.validate(data) if errors.empty?
+      errors = InvalidOutputError::Detail.from(prompt.schema.validate(data), :schema) if errors.empty?
       return [nil, cap(errors)] if errors.any?
 
       result = prompt.schema.build(data, squished: true)
-      [result, prompt.role == :judge ? [] : validator_errors(result)]
+      [result, prompt.role == :judge ? [] : InvalidOutputError::Detail.from(validator_errors(result), :validator)]
     end
 
     private
 
-    # RubyLLM parses structured output itself and leaves the raw string when that fails, so
+    # An adapter may parse structured output itself (RubyLLM does) and leave the raw string when that fails, so
     # content is a Hash on success, or a String/nil when the model refused, was cut off, or
     # ignored the schema.
     def parse(content)
-      return [nil, [UNREPRESENTABLE]] if content.is_a?(String) && !content.valid_encoding?
-      return [nil, ["response was empty"]] if content.nil? || (content.is_a?(String) && content.strip.empty?)
+      return [nil, [parse_error(UNREPRESENTABLE)]] if content.is_a?(String) && !content.valid_encoding?
+      return [nil, [parse_error(EMPTY)]] if content.nil? || (content.is_a?(String) && content.strip.empty?)
 
       data = content.is_a?(String) ? JSON.parse(strip_code_fence(content)) : content
-      representable?(data) ? [data, []] : [nil, [UNREPRESENTABLE]]
+      representable?(data) ? [data, []] : [nil, [parse_error(UNREPRESENTABLE)]]
     rescue JSON::ParserError => e
-      [nil, ["response was not valid JSON#{parse_position(e)}"]]
+      [nil, [parse_error("response was not valid JSON#{parse_position(e)}")]]
     end
 
+    def parse_error(message) = InvalidOutputError::Detail.new(message, :parse)
+
     # JSON.parse turns an out-of-range number such as 1e400 into Infinity, and accepts invalid UTF-8 in a string.
     # Both can satisfy the schema but can't be serialized again (and the deterministic path rejects them), so
     # they are invalid output.
@@ -54,7 +57,8 @@ module Squishling
     def cap(errors)
       return errors if errors.size <= MAX_ERRORS
 
-      errors.first(MAX_ERRORS) + ["... and #{errors.size - MAX_ERRORS} more errors"]
+      notice = "... and #{errors.size - MAX_ERRORS} more errors"
+      errors.first(MAX_ERRORS) + [InvalidOutputError::Detail.new(notice, :squishling)]
     end
 
     # The parser's message quotes a snippet of the response, which must not reach error messages or logs

data/lib/squishling/params.rb

@@ -5,20 +5,19 @@ module Squishling
   module Params
     THINKING_KEYS = %i[effort budget display].freeze
 
-    # Request keys Squishling or RubyLLM own. Provider options are merged into the request last,
-    # overriding RubyLLM's defaults, so these would silently replace the model, the conversation,
-    # or the strict output format.
+    # Request keys Squishling or the adapter own. Provider params are merged into the request alongside what the
+    # adapter builds, so these would silently replace the model, the conversation, or the strict output format.
     RESERVED_KEYS = %i[
       model messages input inputs instructions contents system system_instruction systemInstruction cachedContent
       stream stream_options store include response_format text output_config outputConfig tools tool_choice
-      toolConfig tool_config schema
+      toolConfig tool_config schema functions function_call
     ].freeze
 
     # Some protocols carry the structured-output format (or tool selection) inside a container that is
     # itself allowed, because it also holds ordinary settings (Gemini's topK, Mistral's top_p, ...). RubyLLM
     # deep-merges provider options, so these nested keys would override the strict format just like the
     # top-level ones. Re-check each protocol's render_payload (ruby_llm protocols/*/chat.rb) on every
-    # ruby_llm upgrade.
+    # ruby_llm upgrade (the RubyLLM adapter is where they matter).
     GENERATION_CONFIG_RESERVED_KEYS = %i[
       responseMimeType response_mime_type responseSchema response_schema responseJsonSchema response_json_schema
       response_format tools tool_choice toolConfig tool_config
@@ -37,18 +36,23 @@ module Squishling
       raise ConfigurationError, "#{label} must be a Hash, got #{params.class}" unless params.is_a?(Hash)
 
       params = params.transform_keys(&:to_sym)
-      reserved = params.keys & RESERVED_KEYS
-      if reserved.any?
-        raise ConfigurationError, "#{label}: #{reserved.join(', ')} can't be set through params " \
-                                  "(controlled by Squishling/RubyLLM; use model:/provider:/output_schema)"
-      end
+      reject_reserved!(params, RESERVED_KEYS, label, "; use model:/provider:/output_schema")
       reject_nested_reserved!(params, label)
       params[:thinking] = normalize_thinking(params[:thinking], label) unless params[:thinking].nil?
       params.freeze
     end
 
+    # Raises when params sets any of the keys Squishling owns. hint is appended inside the parenthetical.
+    def reject_reserved!(params, keys, label, hint = "")
+      reserved = params.keys & keys
+      return if reserved.empty?
+
+      raise ConfigurationError, "#{label}: #{reserved.join(', ')} can't be set through params " \
+                                "(controlled by Squishling and the adapter#{hint})"
+    end
+
     # Nested keys are compared as symbols so a string-keyed "responseMimeType" can't slip past the check.
-    # A non-Hash container would replace (not merge into) the one RubyLLM builds, wiping the strict format.
+    # A non-Hash container would replace (not merge into) the one the adapter builds, wiping the strict format.
     # Validated containers are copied and frozen so they can't be mutated after the check.
     def reject_nested_reserved!(params, label)
       nested = NESTED_RESERVED_KEYS.flat_map do |container, reserved|
@@ -71,22 +75,6 @@ module Squishling
       layers.compact.reduce({}) { |merged, layer| merged.merge(layer) }.compact
     end
 
-    # Portable settings use RubyLLM's dedicated setters, which translate them for each provider;
-    # everything else is merged into the provider request as-is.
-    def apply(chat, params)
-      rest = params.except(:temperature, :thinking, :max_output_tokens)
-      chat.with_temperature(params[:temperature]) if params.key?(:temperature)
-      chat.with_max_output_tokens(params[:max_output_tokens]) if params.key?(:max_output_tokens)
-      apply_thinking(chat, params[:thinking]) if params.key?(:thinking)
-      chat.with_provider_options(rest) if rest.any?
-      chat
-    end
-
-    # true: the model's default thinking; false: off; a Hash: { effort:, budget:, display: }.
-    def apply_thinking(chat, thinking)
-      thinking.is_a?(Hash) ? chat.with_thinking(**thinking) : chat.with_thinking(thinking)
-    end
-
     def normalize_thinking(thinking, label)
       return thinking if [true, false].include?(thinking)
 

data/lib/squishling/railtie.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Squishling
+  # Adds the squishling:verify rake task to a Rails app (loaded only when Rails is).
+  class Railtie < ::Rails::Railtie
+    rake_tasks do
+      load File.expand_path("tasks/squishling.rake", __dir__)
+    end
+  end
+end

data/lib/squishling/result.rb

@@ -4,6 +4,17 @@ module Squishling
   # Builds Data classes from a JSON Schema so both the LLM and deterministic paths return
   # the same typed object.
   module Result
+    # Property names a generated class can't take: a Data member of the same name replaces the method, which the
+    # result (or Squishling) needs. result_spec.rb probes every Object, Kernel, and Data method, so a method a newer
+    # Ruby adds can't slip through unreserved.
+    RESERVED_NAMES = %i[
+      squished? squishling_squished []
+      class is_a? kind_of? inspect to_s to_h hash == eql? members deconstruct_keys with dup
+      send public_send
+      __send__ object_id __id__
+      initialize initialize_copy initialize_dup initialize_clone respond_to_missing?
+    ].freeze
+
     # Mixed into every generated result class.
     module Instance
       def initialize(squishling_squished: false, **attrs)
@@ -28,15 +39,9 @@ module Squishling
     module ClassMethods
       def from_h(hash, squished: false)
         hash = hash.transform_keys(&:to_sym)
-        attrs = members.to_h { |member| [member, squishling_fields[member].call(hash[member], squished)] }
+        attrs = members.to_h { |member| [member, @squishling_fields[member].call(hash[member], squished)] }
         new(squishling_squished: squished, **attrs)
       end
-
-      private
-
-      def squishling_fields
-        @squishling_fields ||= @squishling_builder.fields_for(@squishling_schema)
-      end
     end
 
     class << self
@@ -67,21 +72,37 @@ module Squishling
         schema = resolve(schema)
         return nil unless schema.is_a?(Hash) && schema["properties"].is_a?(Hash)
 
-        @classes[schema] ||= begin
-          klass = Data.define(*schema["properties"].keys.map(&:to_sym))
-          klass.include(Instance)
-          klass.extend(ClassMethods)
-          klass.instance_variable_set(:@squishling_builder, self)
-          klass.instance_variable_set(:@squishling_schema, schema)
-          klass
-        end
+        return @classes[schema] if @classes.key?(schema)
+
+        klass = @classes[schema] = define_class(schema)
+        # Memoized first so recursive schemas stop; typing the fields builds nested classes now, so a bad
+        # property name in any object the result types (nested, array items, resolved $refs) raises at schema build.
+        klass.instance_variable_set(:@squishling_fields, fields_for(schema))
+        klass
       end
 
+      private
+
       def fields_for(schema)
         schema["properties"].to_h { |key, property| [key.to_sym, converter_for(property)] }
       end
 
-      private
+      def define_class(schema)
+        klass = Data.define(*property_names(schema))
+        klass.include(Instance)
+        klass.extend(ClassMethods)
+        klass
+      end
+
+      def property_names(schema)
+        names = schema["properties"].keys.map(&:to_sym)
+        clashes = names & RESERVED_NAMES
+        return names if clashes.empty?
+
+        raise ConfigurationError,
+          "output_schema property names clash with methods the result needs: #{clashes.join(', ')}. " \
+          "Rename the field(s) (e.g. `class` to `ticket_class`)"
+      end
 
       def converter_for(property)
         property = resolve(property)

data/lib/squishling/rotation.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Squishling
+  # How many times each squished method has been called with models to rotate between (see ModelPath: steps tied on
+  # priority). The count is the call's turn: each tier of tied models starts that many steps further along, so
+  # successive calls give every model an even share of the first attempts. Counted per class and method, across
+  # threads.
+  module Rotation
+    # class => { method name => calls so far }. Weak keys, so classes replaced by code reloading can be collected.
+    COUNTS = ObjectSpace::WeakKeyMap.new
+    COUNTS_LOCK = Mutex.new
+    private_constant :COUNTS, :COUNTS_LOCK
+
+    class << self
+      # The turn the next call will get, without counting it.
+      def peek(klass, name)
+        COUNTS_LOCK.synchronize { (COUNTS[klass] || {}).fetch(name, 0) }
+      end
+
+      # This call's turn, counting from 0, and counts it.
+      def advance(klass, name)
+        COUNTS_LOCK.synchronize do
+          counts = (COUNTS[klass] ||= {})
+          turn = counts.fetch(name, 0)
+          counts[name] = turn + 1
+          turn
+        end
+      end
+    end
+  end
+end

data/lib/squishling/rspec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "rspec/expectations"
+require "squishling"
+
+# be_squishling_valid: passes when Squishling.verify finds no problems, and fails with its report.
+#   expect(Squishling).to be_squishling_valid   # every squished class (eager load first in Rails)
+#   expect(InvoiceParser).to be_squishling_valid
+#   expect([InvoiceParser, Triage]).to be_squishling_valid
+RSpec::Matchers.define :be_squishling_valid do
+  match do |actual|
+    @report = actual.equal?(Squishling) ? Squishling.verify : Squishling.verify(actual)
+    @report.ok?
+  end
+
+  failure_message { @report.to_s }
+  failure_message_when_negated { "expected Squishling.verify to find problems, but it found none" }
+end

data/lib/squishling/run/messages.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+module Squishling
+  class Run
+    # The follow-up messages a Run sends a model after an output was rejected.
+    module Messages
+      RETRY_HEAD = "Your previous response was rejected:\n- "
+      RETRY_TAIL = "\nRespond again with corrected JSON only."
+      RETURNED = "\n\nA previous attempt at this request returned:\n"
+      REJECTED = "\nIt was rejected:\n- "
+      ESCALATION_TAIL = "\nRespond with corrected JSON only."
+      private_constant :RETRY_HEAD, :RETRY_TAIL, :RETURNED, :REJECTED, :ESCALATION_TAIL
+
+      # What an escalation message carried: the original input, the (capped) rejected output text, and the errors.
+      Escalation = Data.define(:input, :rejected, :errors)
+
+      RETRY_PATTERN = /\A#{Regexp.escape(RETRY_HEAD)}(.*)#{Regexp.escape(RETRY_TAIL)}\z/m
+      ESCALATION_PATTERN = Regexp.new(
+        "\\A(.*?)#{Regexp.escape(RETURNED)}(.*)#{Regexp.escape(REJECTED)}(.*)#{Regexp.escape(ESCALATION_TAIL)}\\z",
+        Regexp::MULTILINE
+      )
+      private_constant :RETRY_PATTERN, :ESCALATION_PATTERN
+
+      class << self
+        # The errors a retry message carries, or nil when text isn't one. The inverse of the message a Run sends, kept
+        # here so the two can't drift apart (the test helpers read what was sent through it). Errors are told apart
+        # by splitting on "\n- ", so one that itself contains that sequence reads back as several.
+        def parse_retry(text)
+          text.match(RETRY_PATTERN)&.then { |match| match[1].split("\n- ") }
+        end
+
+        # What an escalation message carried, or nil when text isn't one.
+        def parse_escalation(text)
+          match = text.match(ESCALATION_PATTERN)
+          match && Escalation.new(input: match[1], rejected: match[2], errors: match[3].split("\n- "))
+        end
+      end
+
+      private
+
+      def retry_message(errors)
+        "#{RETRY_HEAD}#{errors.join("\n- ")}#{RETRY_TAIL}"
+      end
+
+      # The first message to a chat: the input, along with the output rejected before it (if any) unless the step
+      # opts out of seeing it.
+      def fresh_message(step, input, rejected)
+        rejected && step.forward_rejected ? escalation_message(input, *rejected) : input
+      end
+
+      # The first message to a fresh chat after an earlier model's output was rejected. The rejected output is
+      # model-generated and may cross providers, so it is capped at Invoker::MAX_FORWARDED_CHARS.
+      def escalation_message(input, raw, errors)
+        previous = rejected_text(raw)
+        previous = "(an empty response)" if previous.strip.empty? || raw.nil?
+        if previous.length > Invoker::MAX_FORWARDED_CHARS
+          omitted = previous.length - Invoker::MAX_FORWARDED_CHARS
+          previous = "#{previous[0, Invoker::MAX_FORWARDED_CHARS]}... [truncated, #{omitted} more characters]"
+        end
+        "#{input}#{RETURNED}#{previous}#{REJECTED}#{errors.join("\n- ")}#{ESCALATION_TAIL}"
+      end
+
+      # The rejected output as text. Output that was rejected for being unrepresentable in JSON (Infinity, invalid
+      # UTF-8) can't be serialized or measured as is, so it is scrubbed or replaced rather than raising.
+      def rejected_text(raw)
+        return utf8(raw).scrub("?") if raw.is_a?(String)
+
+        JSON.generate(raw)
+      rescue JSON::JSONError
+        "(a response JSON can't represent)"
+      end
+
+      # A binary-tagged string counts every byte as valid, so it is read as UTF-8 before scrubbing.
+      def utf8(text)
+        text.encoding == Encoding::BINARY ? text.dup.force_encoding(Encoding::UTF_8) : text
+      end
+    end
+  end
+end

data/lib/squishling/run/rejections.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Squishling
+  class Run
+    # What a Run does with a reply it rejects: the attempt that follows, the error it raises, the reason it logs, and
+    # the token budget that can end the call before another request. A model that sends back the reply it was just told
+    # was invalid, in the same chat, would do it again, so a repeated reply gives up on the rest of its step's attempts.
+    module Rejections
+      private
+
+      # The number of the attempt after attempt in path, or nil when none is left.
+      def next_attempt(path, attempt, repeated)
+        return (attempt + 1 if attempt < path.size) unless repeated
+
+        skipped = path.drop(attempt).index { |step| step != path[attempt - 1] }
+        attempt + skipped + 1 if skipped
+      end
+
+      def invalid_output(response, errors, attempt, models, source = "LLM")
+        InvalidOutputError.new(errors:, raw: response.content, attempts: attempt, models:, source:,
+          finish_reason: response.finish_reason)
+      end
+
+      # Why an attempt was rejected, for the log: the output check's errors, plus a note when the reply was cut off.
+      def rejection_reason(errors, response)
+        reason = errors.join("; ")
+        response.truncated? ? "#{reason} (#{InvalidOutputError::CUTOFF_NOTE})" : reason
+      end
+
+      # Raises BudgetExhaustedError when the call has spent its token budget (see Budget), before another request.
+      # attempts is how many were made so far, models the ones tried, rejected the last [raw content, errors].
+      def check_budget!(attempts: 0, models: nil, rejected: nil)
+        exhausted = @client.exhausted_budget
+        return unless exhausted
+
+        error = BudgetExhaustedError.new(**exhausted, errors: rejected&.last || [], raw: rejected&.first,
+          attempts: (attempts if attempts.positive?), models: (models.dup if models&.any?))
+        log("stopped, #{error.errors.first}")
+        raise error
+      end
+    end
+  end
+end