squishling 0.5.0 → 0.5.1

data/lib/squishling/configuration.rb

@@ -37,7 +37,8 @@ module Squishling
     # Optional Logger for routing and escalation decisions.
     attr_accessor :logger
 
-    # Optional callable run after each LLM attempt (narrowed by squawk_on) with the raw output (see Squawk), e.g.
+    # Optional callable run after each LLM attempt (narrowed by squawk_on), and once more when a token budget or a
+    # disagreement stops the call, with the raw output (see Squawk), e.g.
     #   ->(output:, metadata:, error:) { Tracer.record(output, metadata, error) }
     attr_reader :squawk
 

data/lib/squishling/errors.rb

@@ -7,10 +7,11 @@ module Squishling
   # missing API key). Never retried and never passed to squish_fallback.
   class ConfigurationError < Error; end
 
-  # The provider answered a request with HTTP 400 (or another 4xx that isn't a credentials, model, or redirect
-  # problem). Still a ConfigurationError: never retried, escalated, or passed to a fallback. The provider's message is
-  # never repeated; the machine-readable status and code its error body carries are, when they are plain tokens
-  # (e.g. "INVALID_ARGUMENT", "invalid_request_error"). Both are nil when the body has none.
+  # The provider answered a request with HTTP 400 (or another 4xx that isn't a credentials, billing, model, redirect,
+  # or too-long-input problem; the adapter decides which, see Adapter::Base#http_failure). Still a ConfigurationError:
+  # never retried, escalated, or passed to a fallback. The provider's message is never repeated; the machine-readable
+  # status and code its error body carries are, when they are plain tokens (e.g. "INVALID_ARGUMENT",
+  # "invalid_request_error"). Both are nil when the body has none.
   class RequestRejectedError < ConfigurationError
     attr_reader :provider_status, :provider_code
 
@@ -130,6 +131,7 @@ module Squishling
   end
 
   # The LLM call itself failed (rate limit, server error, timeout, connection) after the
-  # adapter's own HTTP retries. The original exception is available as #cause.
+  # adapter's own HTTP retries. #cause is the adapter's: a sanitized stand-in (Adapter::ProviderFailure) for a provider
+  # exception that carries a response body, or the transport error itself when it carries none.
   class LLMError < Error; end
 end

data/lib/squishling/invoker.rb

@@ -25,13 +25,14 @@ module Squishling
     Built = Data.define(:prompt, :payload, :purpose)
 
     # Runs the harness (Definition#harness) over a Run of the call: the harness decides how the escalation is
-    # used, and what it returns is checked against the output schema's type.
+    # used, and what it returns is checked against the output schema's type. A call that ends on the token budget
+    # or a disagreement is reported to squawk once here, as the stop no attempt of its own reported.
     def call
       built = build
       harness = @definition.harness
       run = Run.new(definition: @definition, receiver: @receiver, inputs: @inputs, prompt: built.prompt,
         payload: built.payload, purpose: built.purpose, harness:)
-      accepted(harness.klass.new.call(run, **harness.options), run, harness)
+      accepted(call_harness(harness, run), run, harness)
     end
 
     # The first request the call would make (system prompt, output schema, user message), without making it.
@@ -41,6 +42,13 @@ module Squishling
 
     private
 
+    def call_harness(harness, run)
+      harness.klass.new.call(run, **harness.options)
+    rescue BudgetExhaustedError, DisagreementError => e
+      run.report_stop(e)
+      raise
+    end
+
     def build
       purpose = @definition.purpose(@receiver)
       schema = @definition.schema
@@ -70,16 +78,26 @@ module Squishling
     end
 
     # Exceptions are sent as their class and message. Plain JSON would send only the message, and with
-    # json/add/exception loaded it would also send the backtrace, which exposes file paths.
-    def describe(value)
+    # json/add/exception loaded it would also send the backtrace, which exposes file paths. path is the Hashes and
+    # Arrays being walked: one that contains itself is a ConfigurationError rather than a SystemStackError.
+    def describe(value, path = [])
       case value
       when Exception then { class: exception_class_name(value.class), message: value.message }
-      when Hash then value.transform_values { |item| describe(item) }
-      when Array then value.map { |item| describe(item) }
+      when Hash then within(value, path) { |inner| value.transform_values { |item| describe(item, inner) } }
+      when Array then within(value, path) { |inner| value.map { |item| describe(item, inner) } }
       else value
       end
     end
 
+    def within(container, path)
+      if path.any? { |ancestor| ancestor.equal?(container) }
+        raise ConfigurationError, "#{@definition.label}: an argument or context value contains itself, " \
+                                  "so it can't be sent as JSON"
+      end
+
+      yield path + [container]
+    end
+
     # An anonymous error class (Class.new(StandardError)) is named after its closest named ancestor.
     def exception_class_name(klass)
       klass = klass.superclass until klass.name

data/lib/squishling/observer.rb

@@ -33,12 +33,12 @@ module Squishling
         return observer if BUILT_IN.include?(observer)
 
         raise ConfigurationError, "#{where} observer must be one of #{BUILT_IN.map(&:inspect).join(', ')}, " \
-                                  "an object that responds to around, or false, got #{observer.inspect}"
+                                  "an object that responds to around, or false, got #{Appendices.describe(observer)}"
       end
       return observer if observer.respond_to?(:around)
 
       raise ConfigurationError, "#{where} observer must respond to around(call) { ... } (or be false), " \
-                                "got #{observer.inspect}"
+                                "got #{Appendices.describe(observer)}"
     end
 
     # The observer that wraps a request: the configured one, the Coolhand observer when nothing is configured and

data/lib/squishling/params.rb

@@ -9,8 +9,8 @@ module Squishling
     # adapter builds, so these would silently replace the model, the conversation, or the strict output format.
     RESERVED_KEYS = %i[
       model messages input inputs instructions contents system system_instruction systemInstruction cachedContent
-      stream stream_options store include response_format text output_config outputConfig tools tool_choice
-      toolConfig tool_config schema functions function_call
+      cached_content stream stream_options store include response_format text output_config outputConfig tools
+      tool_choice toolConfig tool_config schema functions function_call
     ].freeze
 
     # Some protocols carry the structured-output format (or tool selection) inside a container that is

data/lib/squishling/result.rb

@@ -74,6 +74,7 @@ module Squishling
 
         return @classes[schema] if @classes.key?(schema)
 
+        require_object_type!(schema)
         klass = @classes[schema] = define_class(schema)
         # Memoized first so recursive schemas stop; typing the fields builds nested classes now, so a bad
         # property name in any object the result types (nested, array items, resolved $refs) raises at schema build.
@@ -83,6 +84,18 @@ module Squishling
 
       private
 
+      # `properties` alone constrains objects only, so a schema without `type: "object"` accepts a String or Array
+      # that would then sit in the typed result where a nested result is expected. A raw Hash schema must say so.
+      def require_object_type!(schema)
+        type = schema["type"]
+        return if type == "object" || (type.is_a?(Array) && type.include?("object"))
+
+        raise ConfigurationError,
+          "output_schema has a schema with `properties` whose `type` isn't \"object\" (or missing), so a non-object " \
+          "value would pass validation and end up in the typed result. Set `type: \"object\"` on it " \
+          "(a union such as [\"object\", \"null\"] is also fine)"
+      end
+
       def fields_for(schema)
         schema["properties"].to_h { |key, property| [key.to_sym, converter_for(property)] }
       end

data/lib/squishling/router.rb

@@ -25,9 +25,10 @@ module Squishling
           begin
             frame.phase = :ruby
             value = impl.call
-          rescue NotImplementedError
+          rescue NotImplementedError => e
             # After squish!, the error came from the LLM path (a fallback, a proc), not a missing implementation.
-            raise if frame.handed_off
+            # Otherwise it propagates unless it is the method's own stub (see Wrapper.stub?).
+            raise if frame.handed_off || !Wrapper.stub?(e)
 
             return route_to_llm(frame, definition, "not implemented")
           end

data/lib/squishling/run/reports.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Squishling
+  class Run
+    # What a Run tells the squawk hook (see Squawk): the report of each attempt, and the one final report of a call that
+    # a token budget or a disagreement stopped.
+    module Reports
+      # Set on a stop error once it has been reported (see #report_stop).
+      REPORTED = :@squawk_reported
+      private_constant :REPORTED
+
+      # Reports to squawk the BudgetExhaustedError or DisagreementError that stopped the call: one final report with
+      # the error, for an ending that no attempt of its own reported (the attempt before a budget stop was already
+      # reported, as not final, and a disagreement's samples were both accepted). Nothing was sent, so there is no
+      # output, and the provider and params are unknown (the stop comes from the harness, not from a step).
+      # Internal: Invoker calls it once, whatever the harness, so a custom harness has nothing to do.
+      def report_stop(error)
+        hook = @definition.squawk
+        attempts = error.attempts.to_i
+        return if error.instance_variable_get(REPORTED) || !hook
+        return unless Squawk.report?(@definition.squawk_on, attempts, error, true)
+
+        # Marked first, so an error that comes back through an outer squished call (or a second report_stop) is not
+        # reported again, and a hook that raises doesn't get the same error again either.
+        error.instance_variable_set(REPORTED, true)
+        metadata = squawk_metadata(
+          attempt: attempts, attempts:, final: true, model: error.models&.last, input: @prompt.input,
+          stopped: error.is_a?(BudgetExhaustedError) ? :budget : :disagreement
+        )
+        Squawk.call(hook, output: nil, metadata:, error:)
+      end
+
+      private
+
+      # Runs the observability hook, if any, with this attempt's raw output (nil when the call itself failed),
+      # the error that ended it (nil when it was accepted), and what was asked. role is nil for escalation
+      # attempts, :sample (with the sample's name) or :judge. reask is whether the attempt retried in the same chat,
+      # repeated whether it sent back the reply the attempt before it was rejected for, and final whether no attempt
+      # follows it.
+      def squawk(prompt, path, attempt, response, error, sample: nil, reask: false, repeated: false,
+        final: attempt == path.size)
+        hook = @definition.squawk
+        return unless hook && Squawk.report?(@definition.squawk_on, attempt, error, final)
+
+        step = path[attempt - 1]
+        metadata = squawk_metadata(
+          attempt:, attempts: path.size, final:, repeated:, model: response&.model || step.model,
+          provider: step.provider, params: step.request_params(reask), input: prompt.input, response:,
+          role: sample ? :sample : prompt.role, sample: sample&.name
+        )
+        Squawk.call(hook, output: response&.content, metadata:, error:)
+      end
+
+      def squawk_metadata(attempt:, attempts:, final:, model:, input:, repeated: false, provider: nil, params: nil,
+        response: nil, role: nil, sample: nil, stopped: nil)
+        {
+          label:, attempt:, attempts:, final:, repeated:, model:, provider:, params:, input:,
+          usage: response&.usage, finish_reason: response&.finish_reason, role:, sample:, stopped:
+        }
+      end
+    end
+  end
+end

data/lib/squishling/run.rb

@@ -14,8 +14,9 @@ module Squishling
   class Run
     include Messages
     include Rejections
+    include Reports
 
-    private_constant :Messages, :Rejections
+    private_constant :Messages, :Rejections, :Reports
 
     # One of #samples' outputs, between rounds.
     Sample = Struct.new(:name, :stage, :chat, :rejected, :result, :reask)
@@ -41,7 +42,9 @@ module Squishling
       @produced = {}.compare_by_identity
     end
 
-    # Whether value is a result one of this run's primitives returned (not one built by hand).
+    # Whether value is a result one of this run's primitives returned (not one built by hand). By identity, so it
+    # tells results that are objects (the typed Data of an object schema) apart; a schema that isn't an object
+    # returns the bare JSON value, and an immediate such as 1 or true is the same object in every run.
     def produced?(value)
       @produced.key?(value)
     end
@@ -287,26 +290,6 @@ module Squishling
       @client.chat(step, instructions: prompt.instructions, schema: prompt.schema.llm_schema)
     end
 
-    # Runs the observability hook, if any, with this attempt's raw output (nil when the call itself failed),
-    # the error that ended it (nil when it was accepted), and what was asked. role is nil for escalation
-    # attempts, :sample (with the sample's name) or :judge. reask is whether the attempt retried in the same chat,
-    # repeated whether it sent back the reply the attempt before it was rejected for, and final whether no attempt
-    # follows it.
-    def squawk(prompt, path, attempt, response, error, sample: nil, reask: false, repeated: false,
-      final: attempt == path.size)
-      hook = @definition.squawk
-      return unless hook && Squawk.report?(@definition.squawk_on, attempt, error, final)
-
-      step = path[attempt - 1]
-      metadata = {
-        label:, attempt:, attempts: path.size, final:, repeated:,
-        model: response&.model || step.model, provider: step.provider, params: step.request_params(reask),
-        input: prompt.input, usage: response&.usage, finish_reason: response&.finish_reason,
-        role: sample ? :sample : prompt.role, sample: sample&.name
-      }
-      Squawk.call(hook, output: response&.content, metadata:, error:)
-    end
-
     # following is the attempt that comes next, which is not the next one when a repeated reply skips ahead.
     def log_failure(role, attempt, path, reason, following: attempt + 1, repeated: false)
       step = path[attempt - 1]

data/lib/squishling/schema.rb

@@ -4,8 +4,15 @@ module Squishling
   # Normalizes the supported output-schema forms into a JSON Schema, validates data against it,
   # and builds typed results.
   class Schema
-    CACHE = {}.compare_by_identity
+    # Held weakly by the raw schema (a Schematist class, a Hash), so classes replaced by code reloading can be
+    # collected along with their schemas. A Hash compares by content, which yields the same schema.
+    CACHE = ObjectSpace::WeakKeyMap.new
     CACHE_LOCK = Mutex.new
+    # What building a validator raises for a $ref (or $schema) it can't resolve. json_schemer's default resolver
+    # refuses remote URIs, so none is fetched.
+    REF_ERRORS = [
+      JSONSchemer::UnknownRef, JSONSchemer::InvalidRefResolution, JSONSchemer::InvalidRefPointer, URI::InvalidURIError
+    ].freeze
 
     # Conditional keywords (Schematist's `given` and `dependent`) that providers' strict modes don't support.
     # They're validated locally and kept out of the schema sent to the LLM.
@@ -49,15 +56,15 @@ module Squishling
         "schema" => provider_schema(@json_schema),
         "strict" => true
       }.compact
-      @validator = JSONSchemer.schema(@json_schema)
+      @validator = validator_for(@json_schema)
       # Built now so a property name the result can't take fails here, not on the first response.
       @result_class = Result.build(@json_schema)
     end
 
     def validate(data)
       errors = @validator.validate(data).map { |error| error["error"] }
-      # A raw Hash schema can leave out type: "object"; the typed result is built from a Hash, so anything else is
-      # invalid output rather than a NoMethodError out of the builder.
+      # A raw Hash schema's type can be a union that includes "object" (a bare `properties` is refused at build);
+      # the typed result is built from a Hash, so anything else is invalid output rather than a NoMethodError.
       errors.empty? && result_class && !data.is_a?(Hash) ? ["root must be an object"] : errors
     end
 
@@ -67,6 +74,39 @@ module Squishling
 
     private
 
+    # The json_schemer validator, with every $ref resolved now, so one that points nowhere (a remote URI, a missing
+    # $defs entry) fails at build and in Squishling.verify, not on the first response.
+    def validator_for(json_schema)
+      validator = JSONSchemer.schema(json_schema)
+      resolve_refs!(validator, {}.compare_by_identity)
+      validator
+    rescue *REF_ERRORS => e
+      raise ConfigurationError, "output_schema has a $ref that can't be resolved (#{e.message}); " \
+                                "only references within the schema are supported"
+    end
+
+    # Walks the parsed schema (json_schemer's Schema#parsed and Keyword#ref_schema) and into each target a $ref
+    # resolves to, so a reference inside a subschema kept under any keyword is checked too. seen stops cycles.
+    def resolve_refs!(schema, seen)
+      return if seen.key?(schema)
+
+      seen[schema] = true
+      schema.parsed.each_value do |keyword|
+        resolve_refs!(keyword.ref_schema, seen) if keyword.respond_to?(:ref_schema)
+        subschemas(keyword.parsed).each { |subschema| resolve_refs!(subschema, seen) }
+      end
+    end
+
+    # The Schema objects directly under a parsed keyword: itself, in a list, or in a map of names.
+    def subschemas(parsed)
+      case parsed
+      when JSONSchemer::Schema then [parsed]
+      when Array then parsed.grep(JSONSchemer::Schema)
+      when Hash then parsed.values.grep(JSONSchemer::Schema)
+      else []
+      end
+    end
+
     def to_hash(raw)
       case raw
       when Hash then raw

data/lib/squishling/secret.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Squishling
+  # Holds a credential (an API key, or the headers that may carry one) so a serializer that walks an object's instance
+  # variables can't write it out: to_s, inspect, to_yaml, as_json (so ActiveSupport's instance_values walk too) and
+  # to_json all show "[REDACTED]", and Marshal refuses. The value sits in a closure rather than an instance variable;
+  # #reveal is the only way to read it, and it is for the code that sends the request. Code running in the process can
+  # still call #reveal; this guards against accidental serialization (a logged config, an error tracker, a cache
+  # dump), not against a hostile caller.
+  class Secret
+    REDACTED = "[REDACTED]"
+
+    def initialize(value)
+      @reveal = -> { value }
+      freeze
+    end
+
+    def reveal
+      @reveal.call
+    end
+
+    def inspect
+      REDACTED
+    end
+
+    alias_method :to_s, :inspect
+
+    def encode_with(coder)
+      coder.scalar = REDACTED
+    end
+
+    def as_json(*)
+      REDACTED
+    end
+
+    def to_json(*)
+      REDACTED.to_json
+    end
+
+    def marshal_dump
+      raise TypeError, "Squishling::Secret can't be dumped: it holds a credential"
+    end
+  end
+  private_constant :Secret
+end

data/lib/squishling/squawk.rb

@@ -4,14 +4,16 @@ module Squishling
   # The opt-in observability hook: a callable run after every LLM attempt (or only the rejected and retried
   # ones, see squawk_on:) with the attempt's raw output, so it can be sent somewhere (an error tracker, a
   # tracing tool). It's the one sanctioned way for model output to leave the process; nothing runs unless one
-  # is configured.
+  # is configured. A call that a token budget or a disagreement stops gets one more report, with the error and no
+  # output (metadata's stopped:), since neither has an attempt of its own.
   module Squawk
     KEYWORD_TYPES = %i[key keyreq].freeze
     POSITIONAL_TYPES = %i[req opt].freeze
     KEYWORD_CAPTURE_TYPES = %i[key keyreq keyrest].freeze
     KEYWORDS = %i[output metadata error].freeze
     # The squawk_on: conditions, each answering "does the hook run for this attempt?" from the attempt's 1-based
-    # number (the one its metadata carries), its error (nil when accepted), and whether it is the last one.
+    # number (the one its metadata carries), its error (nil when accepted), and whether it is the last one. A stopped
+    # call's report has an error and is final, so it passes every condition whatever its attempt number (0 or more).
     CONDITIONS = {
       all: ->(_attempt, _error, _final) { true },
       rejections: ->(_attempt, error, _final) { !error.nil? },
@@ -25,7 +27,8 @@ module Squishling
     # Which attempts reach the hook (squawk_on:): one condition or a list of them, or nil to inherit. :all is
     # every attempt, :rejections only the ones that errored, :retries those plus any accepted attempt after the
     # first, and :final the last attempt the escalation allows (metadata's final:), however it ended; an earlier
-    # accepted attempt is not final. A list runs the hook when any of them holds.
+    # accepted attempt is not final. The report of a stopped call is final and has an error, so every
+    # condition lets it through. A list runs the hook when any of them holds.
     def validate_on(value, where)
       return value if value.nil?
 
@@ -48,7 +51,7 @@ module Squishling
       return hook if hook.nil? || hook == false
 
       unless hook.respond_to?(:call)
-        raise ConfigurationError, "#{where} squawk must respond to call (or be false), got #{hook.inspect}"
+        raise ConfigurationError, "#{where} squawk must respond to call (or be false), got #{Appendices.describe(hook)}"
       end
       if positional_only?(parameters(hook))
         raise ConfigurationError,

data/lib/squishling/testing/state.rb

@@ -52,7 +52,8 @@ module Squishling
     end
 
     # What a test asked squawk to record: every attempt's raw output, error, and metadata (role, sample, attempt,
-    # final, usage, ...). Use Testing.with_squawk, which puts the previous hook back.
+    # final, usage, ...), and the report of a call stopped by a token budget or a disagreement (stopped).
+    # Use Testing.with_squawk, which puts the previous hook back.
     class SquawkCapture
       Call = Data.define(:output, :error, :metadata)
 

data/lib/squishling/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Squishling
-  VERSION = "0.5.0"
+  VERSION = "0.5.1"
 end

data/lib/squishling/wrapper.rb

@@ -4,6 +4,24 @@ module Squishling
   # Module prepended to a squishling class. It intercepts squished methods regardless of
   # whether they're defined before or after the `squish` declaration (or at all).
   class Wrapper < Module
+    # Backtrace locations report real paths, so resolve this one too (a symlinked install keeps __FILE__ as given).
+    FILE = File.realpath(__FILE__)
+    private_constant :FILE
+
+    # Raised by the wrapper when a squished method has no implementation at all.
+    class NoImplementation < NotImplementedError; end
+    private_constant :NoImplementation
+
+    # Whether a NotImplementedError is the "no implementation" signal rather than an error from called code: the
+    # wrapper's own, or one raised directly in the method's own frame (`def x = raise NotImplementedError`). The
+    # implementation lambda below sits one frame above the method, so anything raised deeper (a helper, a block,
+    # another object, a `super` into an unsquished parent) has more frames in between. A missing backtrace fails
+    # closed; an exception re-raised or reused keeps the backtrace of its first raise, so a stub that escapes
+    # another squished call (after its `squish!`) still reads as a stub to the caller.
+    def self.stub?(error)
+      error.is_a?(NoImplementation) || error.backtrace_locations&.at(1)&.absolute_path == FILE
+    end
+
     # The method beneath any Squishling wrappers (instance_method resolves to the prepended wrapper first),
     # or nil when the method has no implementation.
     def self.implementation(method)
@@ -20,7 +38,7 @@ module Squishling
           if defined?(super)
             -> { super(*args, **kwargs, &block) }
           else
-            -> { raise NotImplementedError, "#{self.class}##{name} has no implementation" }
+            -> { raise NoImplementation, "#{self.class}##{name} has no implementation" }
           end
 
         Router.dispatch(self, name, args, kwargs, impl, wrapper:)

data/lib/squishling.rb

@@ -9,10 +9,12 @@ require_relative "squishling/errors"
 require_relative "squishling/configuration"
 require_relative "squishling/params"
 require_relative "squishling/named_registry"
+require_relative "squishling/secret"
 require_relative "squishling/adapter/error_body"
 require_relative "squishling/adapter"
 require_relative "squishling/adapters/ruby_llm"
 require_relative "squishling/adapters/openai_options"
+require_relative "squishling/adapters/openai_transport"
 require_relative "squishling/adapters/openai_compatible"
 require_relative "squishling/squawk"
 require_relative "squishling/observer"
@@ -36,6 +38,7 @@ require_relative "squishling/llm_client"
 require_relative "squishling/output_check"
 require_relative "squishling/run/messages"
 require_relative "squishling/run/rejections"
+require_relative "squishling/run/reports"
 require_relative "squishling/run"
 require_relative "squishling/invoker"
 require_relative "squishling/judge"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: squishling
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Michael Carroll
@@ -74,6 +74,7 @@ files:
 - lib/squishling/adapter/error_body.rb
 - lib/squishling/adapters/openai_compatible.rb
 - lib/squishling/adapters/openai_options.rb
+- lib/squishling/adapters/openai_transport.rb
 - lib/squishling/adapters/ruby_llm.rb
 - lib/squishling/appendices.rb
 - lib/squishling/budget.rb
@@ -107,7 +108,9 @@ files:
 - lib/squishling/run.rb
 - lib/squishling/run/messages.rb
 - lib/squishling/run/rejections.rb
+- lib/squishling/run/reports.rb
 - lib/squishling/schema.rb
+- lib/squishling/secret.rb
 - lib/squishling/source.rb
 - lib/squishling/squawk.rb
 - lib/squishling/tasks/squishling.rake