squishling 0.4.0 → 0.5.1

data/lib/squishling/output_check.rb

@@ -5,6 +5,7 @@ module Squishling
   # method's squish_validate check.
   class OutputCheck
     MAX_ERRORS = 20
+    EMPTY = "response was empty"
     UNREPRESENTABLE = "response contained a value JSON can't represent (such as Infinity or invalid UTF-8)"
 
     def initialize(definition, receiver, inputs)
@@ -13,32 +14,34 @@ module Squishling
       @inputs = inputs
     end
 
-    # Parses and validates a response: [typed result, []] when it passes, [nil or result, errors] otherwise.
+    # Parses and validates a response: [typed result, []] when it passes, [nil or result, error Details] otherwise.
     # The judge's role skips squish_validate, which checks the operation's output, not a verdict.
     def call(content, prompt)
       data, errors = parse(content)
-      errors = prompt.schema.validate(data) if errors.empty?
+      errors = InvalidOutputError::Detail.from(prompt.schema.validate(data), :schema) if errors.empty?
       return [nil, cap(errors)] if errors.any?
 
       result = prompt.schema.build(data, squished: true)
-      [result, prompt.role == :judge ? [] : validator_errors(result)]
+      [result, prompt.role == :judge ? [] : InvalidOutputError::Detail.from(validator_errors(result), :validator)]
     end
 
     private
 
-    # RubyLLM parses structured output itself and leaves the raw string when that fails, so
+    # An adapter may parse structured output itself (RubyLLM does) and leave the raw string when that fails, so
     # content is a Hash on success, or a String/nil when the model refused, was cut off, or
     # ignored the schema.
     def parse(content)
-      return [nil, [UNREPRESENTABLE]] if content.is_a?(String) && !content.valid_encoding?
-      return [nil, ["response was empty"]] if content.nil? || (content.is_a?(String) && content.strip.empty?)
+      return [nil, [parse_error(UNREPRESENTABLE)]] if content.is_a?(String) && !content.valid_encoding?
+      return [nil, [parse_error(EMPTY)]] if content.nil? || (content.is_a?(String) && content.strip.empty?)
 
       data = content.is_a?(String) ? JSON.parse(strip_code_fence(content)) : content
-      representable?(data) ? [data, []] : [nil, [UNREPRESENTABLE]]
+      representable?(data) ? [data, []] : [nil, [parse_error(UNREPRESENTABLE)]]
     rescue JSON::ParserError => e
-      [nil, ["response was not valid JSON#{parse_position(e)}"]]
+      [nil, [parse_error("response was not valid JSON#{parse_position(e)}")]]
     end
 
+    def parse_error(message) = InvalidOutputError::Detail.new(message, :parse)
+
     # JSON.parse turns an out-of-range number such as 1e400 into Infinity, and accepts invalid UTF-8 in a string.
     # Both can satisfy the schema but can't be serialized again (and the deterministic path rejects them), so
     # they are invalid output.
@@ -54,7 +57,8 @@ module Squishling
     def cap(errors)
       return errors if errors.size <= MAX_ERRORS
 
-      errors.first(MAX_ERRORS) + ["... and #{errors.size - MAX_ERRORS} more errors"]
+      notice = "... and #{errors.size - MAX_ERRORS} more errors"
+      errors.first(MAX_ERRORS) + [InvalidOutputError::Detail.new(notice, :squishling)]
     end
 
     # The parser's message quotes a snippet of the response, which must not reach error messages or logs

data/lib/squishling/params.rb

@@ -5,20 +5,19 @@ module Squishling
   module Params
     THINKING_KEYS = %i[effort budget display].freeze
 
-    # Request keys Squishling or RubyLLM own. Provider options are merged into the request last,
-    # overriding RubyLLM's defaults, so these would silently replace the model, the conversation,
-    # or the strict output format.
+    # Request keys Squishling or the adapter own. Provider params are merged into the request alongside what the
+    # adapter builds, so these would silently replace the model, the conversation, or the strict output format.
     RESERVED_KEYS = %i[
       model messages input inputs instructions contents system system_instruction systemInstruction cachedContent
-      stream stream_options store include response_format text output_config outputConfig tools tool_choice
-      toolConfig tool_config schema
+      cached_content stream stream_options store include response_format text output_config outputConfig tools
+      tool_choice toolConfig tool_config schema functions function_call
     ].freeze
 
     # Some protocols carry the structured-output format (or tool selection) inside a container that is
     # itself allowed, because it also holds ordinary settings (Gemini's topK, Mistral's top_p, ...). RubyLLM
     # deep-merges provider options, so these nested keys would override the strict format just like the
     # top-level ones. Re-check each protocol's render_payload (ruby_llm protocols/*/chat.rb) on every
-    # ruby_llm upgrade.
+    # ruby_llm upgrade (the RubyLLM adapter is where they matter).
     GENERATION_CONFIG_RESERVED_KEYS = %i[
       responseMimeType response_mime_type responseSchema response_schema responseJsonSchema response_json_schema
       response_format tools tool_choice toolConfig tool_config
@@ -49,11 +48,11 @@ module Squishling
       return if reserved.empty?
 
       raise ConfigurationError, "#{label}: #{reserved.join(', ')} can't be set through params " \
-                                "(controlled by Squishling/RubyLLM#{hint})"
+                                "(controlled by Squishling and the adapter#{hint})"
     end
 
     # Nested keys are compared as symbols so a string-keyed "responseMimeType" can't slip past the check.
-    # A non-Hash container would replace (not merge into) the one RubyLLM builds, wiping the strict format.
+    # A non-Hash container would replace (not merge into) the one the adapter builds, wiping the strict format.
     # Validated containers are copied and frozen so they can't be mutated after the check.
     def reject_nested_reserved!(params, label)
       nested = NESTED_RESERVED_KEYS.flat_map do |container, reserved|
@@ -76,22 +75,6 @@ module Squishling
       layers.compact.reduce({}) { |merged, layer| merged.merge(layer) }.compact
     end
 
-    # Portable settings use RubyLLM's dedicated setters, which translate them for each provider;
-    # everything else is merged into the provider request as-is.
-    def apply(chat, params)
-      rest = params.except(:temperature, :thinking, :max_output_tokens)
-      chat.with_temperature(params[:temperature]) if params.key?(:temperature)
-      chat.with_max_output_tokens(params[:max_output_tokens]) if params.key?(:max_output_tokens)
-      apply_thinking(chat, params[:thinking]) if params.key?(:thinking)
-      chat.with_provider_options(rest) if rest.any?
-      chat
-    end
-
-    # true: the model's default thinking; false: off; a Hash: { effort:, budget:, display: }.
-    def apply_thinking(chat, thinking)
-      thinking.is_a?(Hash) ? chat.with_thinking(**thinking) : chat.with_thinking(thinking)
-    end
-
     def normalize_thinking(thinking, label)
       return thinking if [true, false].include?(thinking)
 

data/lib/squishling/railtie.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Squishling
+  # Adds the squishling:verify rake task to a Rails app (loaded only when Rails is).
+  class Railtie < ::Rails::Railtie
+    rake_tasks do
+      load File.expand_path("tasks/squishling.rake", __dir__)
+    end
+  end
+end

data/lib/squishling/result.rb

@@ -4,6 +4,17 @@ module Squishling
   # Builds Data classes from a JSON Schema so both the LLM and deterministic paths return
   # the same typed object.
   module Result
+    # Property names a generated class can't take: a Data member of the same name replaces the method, which the
+    # result (or Squishling) needs. result_spec.rb probes every Object, Kernel, and Data method, so a method a newer
+    # Ruby adds can't slip through unreserved.
+    RESERVED_NAMES = %i[
+      squished? squishling_squished []
+      class is_a? kind_of? inspect to_s to_h hash == eql? members deconstruct_keys with dup
+      send public_send
+      __send__ object_id __id__
+      initialize initialize_copy initialize_dup initialize_clone respond_to_missing?
+    ].freeze
+
     # Mixed into every generated result class.
     module Instance
       def initialize(squishling_squished: false, **attrs)
@@ -28,15 +39,9 @@ module Squishling
     module ClassMethods
       def from_h(hash, squished: false)
         hash = hash.transform_keys(&:to_sym)
-        attrs = members.to_h { |member| [member, squishling_fields[member].call(hash[member], squished)] }
+        attrs = members.to_h { |member| [member, @squishling_fields[member].call(hash[member], squished)] }
         new(squishling_squished: squished, **attrs)
       end
-
-      private
-
-      def squishling_fields
-        @squishling_fields ||= @squishling_builder.fields_for(@squishling_schema)
-      end
     end
 
     class << self
@@ -67,21 +72,50 @@ module Squishling
         schema = resolve(schema)
         return nil unless schema.is_a?(Hash) && schema["properties"].is_a?(Hash)
 
-        @classes[schema] ||= begin
-          klass = Data.define(*schema["properties"].keys.map(&:to_sym))
-          klass.include(Instance)
-          klass.extend(ClassMethods)
-          klass.instance_variable_set(:@squishling_builder, self)
-          klass.instance_variable_set(:@squishling_schema, schema)
-          klass
-        end
+        return @classes[schema] if @classes.key?(schema)
+
+        require_object_type!(schema)
+        klass = @classes[schema] = define_class(schema)
+        # Memoized first so recursive schemas stop; typing the fields builds nested classes now, so a bad
+        # property name in any object the result types (nested, array items, resolved $refs) raises at schema build.
+        klass.instance_variable_set(:@squishling_fields, fields_for(schema))
+        klass
+      end
+
+      private
+
+      # `properties` alone constrains objects only, so a schema without `type: "object"` accepts a String or Array
+      # that would then sit in the typed result where a nested result is expected. A raw Hash schema must say so.
+      def require_object_type!(schema)
+        type = schema["type"]
+        return if type == "object" || (type.is_a?(Array) && type.include?("object"))
+
+        raise ConfigurationError,
+          "output_schema has a schema with `properties` whose `type` isn't \"object\" (or missing), so a non-object " \
+          "value would pass validation and end up in the typed result. Set `type: \"object\"` on it " \
+          "(a union such as [\"object\", \"null\"] is also fine)"
       end
 
       def fields_for(schema)
         schema["properties"].to_h { |key, property| [key.to_sym, converter_for(property)] }
       end
 
-      private
+      def define_class(schema)
+        klass = Data.define(*property_names(schema))
+        klass.include(Instance)
+        klass.extend(ClassMethods)
+        klass
+      end
+
+      def property_names(schema)
+        names = schema["properties"].keys.map(&:to_sym)
+        clashes = names & RESERVED_NAMES
+        return names if clashes.empty?
+
+        raise ConfigurationError,
+          "output_schema property names clash with methods the result needs: #{clashes.join(', ')}. " \
+          "Rename the field(s) (e.g. `class` to `ticket_class`)"
+      end
 
       def converter_for(property)
         property = resolve(property)

data/lib/squishling/rotation.rb

@@ -12,6 +12,11 @@ module Squishling
     private_constant :COUNTS, :COUNTS_LOCK
 
     class << self
+      # The turn the next call will get, without counting it.
+      def peek(klass, name)
+        COUNTS_LOCK.synchronize { (COUNTS[klass] || {}).fetch(name, 0) }
+      end
+
       # This call's turn, counting from 0, and counts it.
       def advance(klass, name)
         COUNTS_LOCK.synchronize do

data/lib/squishling/router.rb

@@ -25,9 +25,10 @@ module Squishling
           begin
             frame.phase = :ruby
             value = impl.call
-          rescue NotImplementedError
+          rescue NotImplementedError => e
             # After squish!, the error came from the LLM path (a fallback, a proc), not a missing implementation.
-            raise if frame.handed_off
+            # Otherwise it propagates unless it is the method's own stub (see Wrapper.stub?).
+            raise if frame.handed_off || !Wrapper.stub?(e)
 
             return route_to_llm(frame, definition, "not implemented")
           end

data/lib/squishling/rspec.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "rspec/expectations"
+require "squishling"
+
+# be_squishling_valid: passes when Squishling.verify finds no problems, and fails with its report.
+#   expect(Squishling).to be_squishling_valid   # every squished class (eager load first in Rails)
+#   expect(InvoiceParser).to be_squishling_valid
+#   expect([InvoiceParser, Triage]).to be_squishling_valid
+RSpec::Matchers.define :be_squishling_valid do
+  match do |actual|
+    @report = actual.equal?(Squishling) ? Squishling.verify : Squishling.verify(actual)
+    @report.ok?
+  end
+
+  failure_message { @report.to_s }
+  failure_message_when_negated { "expected Squishling.verify to find problems, but it found none" }
+end

data/lib/squishling/run/messages.rb

@@ -4,10 +4,48 @@ module Squishling
   class Run
     # The follow-up messages a Run sends a model after an output was rejected.
     module Messages
+      RETRY_HEAD = "Your previous response was rejected:\n- "
+      RETRY_TAIL = "\nRespond again with corrected JSON only."
+      RETURNED = "\n\nA previous attempt at this request returned:\n"
+      REJECTED = "\nIt was rejected:\n- "
+      ESCALATION_TAIL = "\nRespond with corrected JSON only."
+      private_constant :RETRY_HEAD, :RETRY_TAIL, :RETURNED, :REJECTED, :ESCALATION_TAIL
+
+      # What an escalation message carried: the original input, the (capped) rejected output text, and the errors.
+      Escalation = Data.define(:input, :rejected, :errors)
+
+      RETRY_PATTERN = /\A#{Regexp.escape(RETRY_HEAD)}(.*)#{Regexp.escape(RETRY_TAIL)}\z/m
+      ESCALATION_PATTERN = Regexp.new(
+        "\\A(.*?)#{Regexp.escape(RETURNED)}(.*)#{Regexp.escape(REJECTED)}(.*)#{Regexp.escape(ESCALATION_TAIL)}\\z",
+        Regexp::MULTILINE
+      )
+      private_constant :RETRY_PATTERN, :ESCALATION_PATTERN
+
+      class << self
+        # The errors a retry message carries, or nil when text isn't one. The inverse of the message a Run sends, kept
+        # here so the two can't drift apart (the test helpers read what was sent through it). Errors are told apart
+        # by splitting on "\n- ", so one that itself contains that sequence reads back as several.
+        def parse_retry(text)
+          text.match(RETRY_PATTERN)&.then { |match| match[1].split("\n- ") }
+        end
+
+        # What an escalation message carried, or nil when text isn't one.
+        def parse_escalation(text)
+          match = text.match(ESCALATION_PATTERN)
+          match && Escalation.new(input: match[1], rejected: match[2], errors: match[3].split("\n- "))
+        end
+      end
+
       private
 
       def retry_message(errors)
-        "Your previous response was rejected:\n- #{errors.join("\n- ")}\nRespond again with corrected JSON only."
+        "#{RETRY_HEAD}#{errors.join("\n- ")}#{RETRY_TAIL}"
+      end
+
+      # The first message to a chat: the input, along with the output rejected before it (if any) unless the step
+      # opts out of seeing it.
+      def fresh_message(step, input, rejected)
+        rejected && step.forward_rejected ? escalation_message(input, *rejected) : input
       end
 
       # The first message to a fresh chat after an earlier model's output was rejected. The rejected output is
@@ -19,8 +57,7 @@ module Squishling
           omitted = previous.length - Invoker::MAX_FORWARDED_CHARS
           previous = "#{previous[0, Invoker::MAX_FORWARDED_CHARS]}... [truncated, #{omitted} more characters]"
         end
-        "#{input}\n\nA previous attempt at this request returned:\n#{previous}\n" \
-          "It was rejected:\n- #{errors.join("\n- ")}\nRespond with corrected JSON only."
+        "#{input}#{RETURNED}#{previous}#{REJECTED}#{errors.join("\n- ")}#{ESCALATION_TAIL}"
       end
 
       # The rejected output as text. Output that was rejected for being unrepresentable in JSON (Infinity, invalid

data/lib/squishling/run/rejections.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Squishling
+  class Run
+    # What a Run does with a reply it rejects: the attempt that follows, the error it raises, the reason it logs, and
+    # the token budget that can end the call before another request. A model that sends back the reply it was just told
+    # was invalid, in the same chat, would do it again, so a repeated reply gives up on the rest of its step's attempts.
+    module Rejections
+      private
+
+      # The number of the attempt after attempt in path, or nil when none is left.
+      def next_attempt(path, attempt, repeated)
+        return (attempt + 1 if attempt < path.size) unless repeated
+
+        skipped = path.drop(attempt).index { |step| step != path[attempt - 1] }
+        attempt + skipped + 1 if skipped
+      end
+
+      def invalid_output(response, errors, attempt, models, source = "LLM")
+        InvalidOutputError.new(errors:, raw: response.content, attempts: attempt, models:, source:,
+          finish_reason: response.finish_reason)
+      end
+
+      # Why an attempt was rejected, for the log: the output check's errors, plus a note when the reply was cut off.
+      def rejection_reason(errors, response)
+        reason = errors.join("; ")
+        response.truncated? ? "#{reason} (#{InvalidOutputError::CUTOFF_NOTE})" : reason
+      end
+
+      # Raises BudgetExhaustedError when the call has spent its token budget (see Budget), before another request.
+      # attempts is how many were made so far, models the ones tried, rejected the last [raw content, errors].
+      def check_budget!(attempts: 0, models: nil, rejected: nil)
+        exhausted = @client.exhausted_budget
+        return unless exhausted
+
+        error = BudgetExhaustedError.new(**exhausted, errors: rejected&.last || [], raw: rejected&.first,
+          attempts: (attempts if attempts.positive?), models: (models.dup if models&.any?))
+        log("stopped, #{error.errors.first}")
+        raise error
+      end
+    end
+  end
+end

data/lib/squishling/run/reports.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Squishling
+  class Run
+    # What a Run tells the squawk hook (see Squawk): the report of each attempt, and the one final report of a call that
+    # a token budget or a disagreement stopped.
+    module Reports
+      # Set on a stop error once it has been reported (see #report_stop).
+      REPORTED = :@squawk_reported
+      private_constant :REPORTED
+
+      # Reports to squawk the BudgetExhaustedError or DisagreementError that stopped the call: one final report with
+      # the error, for an ending that no attempt of its own reported (the attempt before a budget stop was already
+      # reported, as not final, and a disagreement's samples were both accepted). Nothing was sent, so there is no
+      # output, and the provider and params are unknown (the stop comes from the harness, not from a step).
+      # Internal: Invoker calls it once, whatever the harness, so a custom harness has nothing to do.
+      def report_stop(error)
+        hook = @definition.squawk
+        attempts = error.attempts.to_i
+        return if error.instance_variable_get(REPORTED) || !hook
+        return unless Squawk.report?(@definition.squawk_on, attempts, error, true)
+
+        # Marked first, so an error that comes back through an outer squished call (or a second report_stop) is not
+        # reported again, and a hook that raises doesn't get the same error again either.
+        error.instance_variable_set(REPORTED, true)
+        metadata = squawk_metadata(
+          attempt: attempts, attempts:, final: true, model: error.models&.last, input: @prompt.input,
+          stopped: error.is_a?(BudgetExhaustedError) ? :budget : :disagreement
+        )
+        Squawk.call(hook, output: nil, metadata:, error:)
+      end
+
+      private
+
+      # Runs the observability hook, if any, with this attempt's raw output (nil when the call itself failed),
+      # the error that ended it (nil when it was accepted), and what was asked. role is nil for escalation
+      # attempts, :sample (with the sample's name) or :judge. reask is whether the attempt retried in the same chat,
+      # repeated whether it sent back the reply the attempt before it was rejected for, and final whether no attempt
+      # follows it.
+      def squawk(prompt, path, attempt, response, error, sample: nil, reask: false, repeated: false,
+        final: attempt == path.size)
+        hook = @definition.squawk
+        return unless hook && Squawk.report?(@definition.squawk_on, attempt, error, final)
+
+        step = path[attempt - 1]
+        metadata = squawk_metadata(
+          attempt:, attempts: path.size, final:, repeated:, model: response&.model || step.model,
+          provider: step.provider, params: step.request_params(reask), input: prompt.input, response:,
+          role: sample ? :sample : prompt.role, sample: sample&.name
+        )
+        Squawk.call(hook, output: response&.content, metadata:, error:)
+      end
+
+      def squawk_metadata(attempt:, attempts:, final:, model:, input:, repeated: false, provider: nil, params: nil,
+        response: nil, role: nil, sample: nil, stopped: nil)
+        {
+          label:, attempt:, attempts:, final:, repeated:, model:, provider:, params:, input:,
+          usage: response&.usage, finish_reason: response&.finish_reason, role:, sample:, stopped:
+        }
+      end
+    end
+  end
+end