squishling 0.1.0 → 0.3.0

data/lib/squishling/definition.rb

@@ -9,38 +9,47 @@ module Squishling
 
     attr_reader :klass, :name, :call_context
 
-    def initialize(klass:, name:, instructions: nil, append_instructions: nil, output_schema: nil, model: nil,
-      provider: nil, params: nil, predicate: nil, fallback: nil, call_context: {})
+    def initialize(klass:, name:, purpose: nil, append_to_purpose: nil, output_schema: nil, model: nil,
+      provider: nil, params: nil, harness: nil, predicate: nil, fallback: nil, validator: nil, squawk: nil,
+      call_context: {})
       @klass = klass
       @name = name
-      @instructions = instructions
-      @append_instructions = append_instructions
+      @purpose = purpose
+      @append_to_purpose = append_to_purpose
       @output_schema = output_schema
       @model = model
       @provider = provider
       @params = params
+      @harness = harness
       @predicate = predicate
       @fallback = fallback
+      @validator = validator
+      @squawk = squawk
       @call_context = call_context
     end
 
-    # This definition with one call's overrides on top. The output schema and predicate can't be overridden:
-    # the call must still return the method's result type.
-    def for_call(instructions: nil, append_instructions: nil, context: nil, model: nil, provider: nil, params: nil)
-      raise ConfigurationError, "#{label}: squish! provider: needs a model:" if provider && !model
+    # This definition with one call's overrides on top. The output schema, predicate, fallback, and validator
+    # can't be overridden: the call must still return the method's result type.
+    def for_call(purpose: nil, append_to_purpose: nil, context: nil, model: nil, escalation: nil,
+      provider: nil, params: nil, harness: nil)
+      call_path = ModelPath.declare(model, escalation, "#{label} squish!")
+      raise ConfigurationError, "#{label}: squish! provider: needs a model: or escalation:" if provider && !call_path
       unless context.nil? || (context.is_a?(Hash) && context.each_key.all?(NAME_KEY))
         raise ConfigurationError, "#{label}: squish! context: must be a Hash with String or Symbol keys"
       end
 
-      appended = Appendices.normalize(append_instructions, "#{label} squish!") unless append_instructions.nil?
+      appended = Appendices.normalize(append_to_purpose, "#{label} squish!") unless append_to_purpose.nil?
       call_params = params && Params.normalize(params, "#{label} squish! params")
+      call_harness = Harness.normalize(harness, "#{label} squish!") unless harness.nil?
       self.class.new(
         klass:, name:, output_schema: @output_schema, predicate: @predicate, fallback: @fallback,
-        instructions: instructions || @instructions,
-        append_instructions: [*@append_instructions, *appended],
-        model: model || @model, provider: model ? provider : @provider,
+        validator: @validator, squawk: @squawk,
+        purpose: purpose || @purpose,
+        append_to_purpose: [*@append_to_purpose, *appended],
+        model: call_path || @model, provider: call_path ? provider : @provider,
         # merge, not Params.resolve: a nil at the method level must still unset the class's key.
         params: call_params ? (@params || {}).merge(call_params) : @params,
+        harness: call_harness || @harness,
         call_context: @call_context.merge((context || {}).transform_keys(&:to_sym))
       )
     end
@@ -49,17 +58,17 @@ module Squishling
       "#{klass}##{name}"
     end
 
-    # The system prompt: the instructions, then each append_instructions section.
-    def instructions(receiver)
-      value = @instructions || klass.instructions
+    # The system prompt: the purpose, then each append_to_purpose section.
+    def purpose(receiver)
+      value = @purpose || klass.purpose
       value = receiver.instance_exec(&value) if value.is_a?(Proc)
       return value if value.nil? || value.empty?
 
-      [value, *Appendices.render(append_instructions, receiver, label)].join("\n\n")
+      [value, *Appendices.render(append_to_purpose, receiver, label)].join("\n\n")
     end
 
-    def append_instructions
-      Appendices.resolve(klass.squishling_append_instructions + (@append_instructions || []))
+    def append_to_purpose
+      Appendices.resolve(klass.squishling_append_to_purpose + (@append_to_purpose || []))
     end
 
     def schema
@@ -67,20 +76,20 @@ module Squishling
       raw && Schema.for(raw)
     end
 
-    def model
-      model_and_provider.first
+    # One ModelPath::Step per attempt, from the first level (method, class, or config) that declares a model
+    # or escalation, or a single attempt on RubyLLM's default model. A provider travels with the model declared
+    # at the same level, so a per-method Anthropic model never inherits a class-level OpenAI provider.
+    def escalation_path
+      config = Squishling.config
+      entries, provider = [[@model, @provider], [klass.squishling_model_path, klass.squishling_provider],
+                           [config.default_model_path, config.default_provider]].find(&:first)
+      entries ||= [{ model: nil, attempts: 1, forward_rejected: true }]
+      ModelPath.steps(entries, provider:, params:)
     end
 
-    def provider
-      model_and_provider.last
-    end
-
-    # A provider travels with the model declared at the same level (method, class, or config),
-    # so a per-method Anthropic model never inherits a class-level OpenAI provider.
-    def model_and_provider
-      config = Squishling.config
-      [[@model, @provider], [klass.squishling_model, klass.squishling_provider],
-       [config.default_model, config.default_provider]].find(&:first) || [nil, nil]
+    # How the escalation is used (see Harness): the method's, the class's, the configured default, or :escalation.
+    def harness
+      @harness || klass.squishling_harness || Squishling.config.default_harness || Harness::DEFAULT
     end
 
     # Generation params: config defaults, overridden key by key by the class, then by the method.
@@ -88,6 +97,17 @@ module Squishling
       Params.resolve(Squishling.config.default_params, klass.squishling_params, @params)
     end
 
+    # Extra output checks run on schema-valid LLM results (see ClassMethods#squish_validate).
+    def validator
+      @validator || klass.squishling_validator
+    end
+
+    # The observability hook: the method's, else the class's, else the configured one. `false` at a level
+    # silences the levels above it.
+    def squawk
+      [@squawk, klass.squishling_squawk, Squishling.config.squawk].find { |hook| !hook.nil? } || nil
+    end
+
     def context_names
       klass.squishling_context_names
     end
@@ -111,43 +131,62 @@ module Squishling
       coerce(receiver.instance_exec(e, **inputs, &handler))
     end
 
-    # Deterministic return values: hashes are validated and turned into the typed result;
-    # anything else (including an already-built result) passes through.
+    # Deterministic and fallback return values: with an output_schema, every value is validated and
+    # typed like an LLM result. Without one, the value passes through untouched.
     def coerce(value)
-      value.is_a?(Hash) && schema ? build_result(value, squished: false) : value
+      schema ? build_result(value, squished: false) : value
     end
 
+    # A result of this schema's own class passes through; any other result is re-validated via to_h.
     def build_result(attrs, squished:)
       raise ConfigurationError, "#{label} has no output_schema" unless schema
-      return attrs if attrs.is_a?(Result::Instance)
+      return attrs if schema.result_class && attrs.is_a?(schema.result_class)
 
-      data = Schema.jsonify(attrs)
-      errors = schema.validate(data)
+      attrs = attrs.to_h if attrs.is_a?(Result::Instance)
+      data, errors = jsonify_return(attrs)
+      errors = schema.validate(data) if errors.empty?
       raise InvalidOutputError.new(errors:, raw: attrs, source: "Deterministic") if errors.any?
 
       schema.build(data, squished:)
     end
 
-    # Map positional and keyword arguments onto the original method's parameter names.
+    # Map positional and keyword arguments onto the original method's parameter names. A positional with no
+    # name (a destructuring parameter, or one beyond the declared parameters) is sent as arg0, arg1, ...
     def bind_arguments(args, kwargs)
       positional = args.dup
       bound = {}
+      unnamed = 0
 
       parameters.each do |type, param|
         case type
         when :req, :opt
-          bound[param] = positional.shift unless positional.empty? || param.nil?
+          next if positional.empty?
+
+          value = positional.shift
+          if param
+            bound[param] = value
+          else
+            bound[:"arg#{unnamed}"] = value
+            unnamed += 1
+          end
         when :rest
           bound[param.nil? || param == :* ? :args : param] = positional.shift(positional.size)
         end
       end
-      positional.each_with_index { |value, index| bound[:"arg#{index}"] = value }
+      positional.each_with_index { |value, index| bound[:"arg#{unnamed + index}"] = value }
 
       bound.merge(kwargs)
     end
 
     private
 
+    # Values JSON can't represent (NaN, Infinity, cycles) are invalid output, not a raw JSON error.
+    def jsonify_return(attrs)
+      [Schema.jsonify(attrs), []]
+    rescue JSON::GeneratorError, JSON::NestingError => e
+      [nil, [e.message]]
+    end
+
     # The implementation's parameters, beneath the prepended wrappers.
     def parameters
       Wrapper.implementation(klass.instance_method(name))&.parameters || []

data/lib/squishling/errors.rb

@@ -3,20 +3,44 @@
 module Squishling
   class Error < StandardError; end
 
-  # A programming or setup mistake (missing instructions/schema, non-strict schema, unknown model,
+  # A programming or setup mistake (missing purpose/schema, non-strict schema, unknown model,
   # missing API key). Never retried and never passed to squish_fallback.
   class ConfigurationError < Error; end
 
-  # Output that didn't match the schema: from the LLM after all retries, or from the deterministic path.
+  # Output that didn't match the schema (or a squish_validate check): from the LLM after every attempt in the
+  # escalation, or from the deterministic path.
   class InvalidOutputError < Error
-    attr_reader :errors, :raw, :attempts
+    # models: the model ids tried, one per attempt (nil entries mean RubyLLM's default model).
+    attr_reader :errors, :raw, :attempts, :models
 
-    def initialize(errors:, raw: nil, source: "LLM", attempts: nil)
+    def initialize(errors:, raw: nil, source: "LLM", attempts: nil, models: nil)
       @errors = errors
       @raw = raw
       @attempts = attempts
+      @models = models
       tries = attempts ? " after #{attempts} attempt#{'s' unless attempts == 1}" : ""
-      super("#{source} output did not match the output schema#{tries}: #{errors.join('; ')}")
+      super("#{source} output was invalid#{tries}: #{errors.join('; ')}")
+    end
+  end
+
+  # A sampling harness's samples (squishsum or ensemble) were valid but didn't agree, and either no judge was declared
+  # (verdict nil) or the judge rejected both (verdict :neither, with its reason). candidates holds the two
+  # typed results, so a squish_fallback can still return one of them, and raw both as Hashes. models names the
+  # model behind each role (sample a, sample b, then the judge if one ran), not one entry per attempt; attempts
+  # is nil.
+  class DisagreementError < InvalidOutputError
+    attr_reader :candidates, :verdict, :reason
+
+    # detail: is text Squishling wrote itself (a judgment model's choice and probability) and is added to the
+    # message. A chat judge's reason is model-written and can quote the input, so it stays out of the message
+    # (and the logs); read it from #reason.
+    def initialize(candidates:, verdict: nil, reason: nil, detail: nil, models: nil)
+      @candidates = candidates
+      @verdict = verdict
+      @reason = reason
+      error = verdict ? "the judge rejected both samples" : "the two samples disagreed"
+      error += " (#{detail})" if detail
+      super(errors: [error], raw: candidates.map(&:to_h), models:)
     end
   end
 

data/lib/squishling/harness.rb

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module Squishling
+  # How a squished call uses its escalation. Declared as a type, or a Hash with the type and its options:
+  #   harness: :squishsum
+  #   harness: { type: :judged_squishsum, judge: "claude-opus-5-5", compare: ->(a, b, **) { a.team == b.team } }
+  # :escalation (the default) tries each attempt in order until an output passes. :squishsum asks the first
+  # escalation step twice, concurrently, and accepts the output only when both samples agree. :ensemble asks
+  # the first and second escalation steps once each instead (a cross-model check). :judged_squishsum and
+  # :judged_ensemble hand disagreeing samples to a judge that picks one or rejects both.
+  class Harness
+    TYPES = %i[escalation squishsum judged_squishsum ensemble judged_ensemble].freeze
+    ENSEMBLE_TYPES = %i[ensemble judged_ensemble].freeze
+    JUDGED_TYPES = %i[judged_squishsum judged_ensemble].freeze
+    KEYS = %i[type judge judge_instructions compare].freeze
+
+    # A judge is one step: a model name or a Hash with the escalation step keys (except order:), plus
+    # type: (:chat, the default, or :judgment for a System One decision model through RubyLLM.judge) and,
+    # for :judgment, min_confidence: (the probability the chosen candidate needs).
+    JUDGE_TYPES = %i[chat judgment].freeze
+    JUDGE_KEYS = %i[type min_confidence].freeze
+    # Request keys RubyLLM's judgment protocols own (model and input are already reserved params).
+    JUDGMENT_RESERVED_PARAMS = %i[state questions].freeze
+    DEFAULT_MIN_CONFIDENCE = 0.8
+
+    DEFAULT_JUDGE_INSTRUCTIONS = <<~TEXT.strip
+      You are an impartial judge. Two independent attempts at the same operation returned different outputs,
+      candidate "a" and candidate "b". Both already match the required output format, so judge only their
+      content. You are given the operation's purpose and its input. Decide which candidate correctly and
+      faithfully carries out the purpose for this input. If both do (for example, they differ only in
+      wording), choose either one, preferring "a". Choose "neither" only when both are wrong or you can't tell
+      whether either is correct. Never combine them or invent a third answer.
+    TEXT
+
+    attr_reader :type, :judge, :judge_instructions, :compare
+
+    class << self
+      def normalize(value, label)
+        case value
+        when Symbol, String then new(type: type_for(value, label))
+        when Hash then from_hash(value, label)
+        else
+          raise ConfigurationError, "#{label}: harness must be one of #{TYPES.join(', ')} or a Hash with type:, " \
+                                    "got #{value.inspect}"
+        end
+      end
+
+      private
+
+      def from_hash(hash, label)
+        hash = hash.transform_keys(&:to_sym)
+        unknown = hash.keys - KEYS
+        raise ConfigurationError, "#{label}: unknown harness option(s) #{unknown.join(', ')}" if unknown.any?
+        raise ConfigurationError, "#{label}: a harness Hash needs type:" unless hash.key?(:type)
+
+        type = type_for(hash[:type], label)
+        check_options!(type, hash, label)
+        new(type:, judge: hash[:judge].nil? ? nil : normalize_judge(hash[:judge], label),
+          judge_instructions: hash[:judge_instructions], compare: hash[:compare])
+      end
+
+      def type_for(value, label)
+        type = value.to_s.to_sym if value.is_a?(Symbol) || value.is_a?(String)
+        return type if TYPES.include?(type)
+
+        raise ConfigurationError, "#{label}: unknown harness #{value.inspect} (use #{TYPES.join(', ')})"
+      end
+
+      def check_options!(type, hash, label)
+        judge_options = %i[judge judge_instructions].select { |key| hash.key?(key) }
+        if judge_options.any? && !JUDGED_TYPES.include?(type)
+          options = judge_options.map { |key| "#{key}:" }.join(", ")
+          raise ConfigurationError,
+            "#{label}: #{options} can only be used with the #{JUDGED_TYPES.join(' and ')} harnesses"
+        end
+        if hash.key?(:compare) && type == :escalation
+          raise ConfigurationError, "#{label}: compare: only applies to the sampling harnesses (squishsum and ensemble)"
+        end
+        unless hash[:compare].nil? || hash[:compare].is_a?(Proc)
+          raise ConfigurationError, "#{label}: compare: must be a Proc, got #{hash[:compare].class}"
+        end
+
+        instructions = hash[:judge_instructions]
+        return if instructions.nil? || instructions.is_a?(Proc)
+        return if instructions.is_a?(String) && !instructions.strip.empty?
+
+        raise ConfigurationError, "#{label}: judge_instructions: must be a non-blank String or a Proc"
+      end
+
+      # The judge as a normalized escalation step (see ModelPath), plus its type and min_confidence.
+      def normalize_judge(judge, label)
+        judge_label = "#{label} judge"
+        return ModelPath.normalize_step(judge, judge_label).merge(type: :chat) unless judge.is_a?(Hash)
+
+        judge = judge.transform_keys(&:to_sym)
+        raise ConfigurationError, "#{judge_label}: a judge is one step, so it takes no order:" if judge.key?(:order)
+
+        type = judge.fetch(:type, :chat)
+        type = type.to_sym if type.is_a?(String)
+        unless JUDGE_TYPES.include?(type)
+          raise ConfigurationError,
+            "#{judge_label}: type: must be one of #{JUDGE_TYPES.join(', ')}, got #{type.inspect}"
+        end
+
+        if type == :chat && judge.key?(:min_confidence)
+          raise ConfigurationError, "#{judge_label}: min_confidence: only applies to type: :judgment judges"
+        end
+
+        step = ModelPath.normalize_step(judge.except(*JUDGE_KEYS), judge_label).merge(type:)
+        return step unless type == :judgment
+
+        reserved = (step[:params] || {}).keys & JUDGMENT_RESERVED_PARAMS
+        if reserved.any?
+          raise ConfigurationError, "#{judge_label}: #{reserved.join(', ')} can't be set through params " \
+                                    "(controlled by Squishling/RubyLLM)"
+        end
+
+        step.merge(min_confidence: min_confidence(judge, judge_label))
+      end
+
+      def min_confidence(judge, label)
+        value = judge.fetch(:min_confidence, DEFAULT_MIN_CONFIDENCE)
+        return value.to_f if value.is_a?(Numeric) && value.between?(0, 1)
+
+        raise ConfigurationError, "#{label}: min_confidence: must be a number from 0 to 1, got #{value.inspect}"
+      end
+    end
+
+    def initialize(type:, judge: nil, judge_instructions: nil, compare: nil)
+      @type = type
+      @judge = judge
+      @judge_instructions = judge_instructions
+      @compare = compare
+      freeze
+    end
+
+    DEFAULT = new(type: :escalation)
+
+    # Whether the call asks two samples (squishsum and ensemble harnesses) rather than walking the escalation.
+    def squishsum?
+      type != :escalation
+    end
+
+    # Whether the two samples come from the first two escalation steps instead of both from the first.
+    def ensemble?
+      ENSEMBLE_TYPES.include?(type)
+    end
+
+    def judged?
+      JUDGED_TYPES.include?(type)
+    end
+  end
+end