squishling 0.3.0 → 0.5.0

data/lib/squishling/harness/base.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module Squishling
+  class Harness
+    # The base of every harness: the built-ins, and your own. Subclass it, declare the options it takes, define
+    # call, and register it (Squishling.register_harness):
+    #
+    #   class Tiebreaker < Squishling::Harness::Base
+    #     option :tiebreaker, :step, required: true
+    #
+    #     def call(run, tiebreaker:)
+    #       first, second = run.samples(run.stages.first(2))
+    #       run.agree?(first, second) ? first : run.escalate(tiebreaker)
+    #     end
+    #   end
+    #   Squishling.register_harness(:tiebreaker, Tiebreaker)
+    #
+    # call gets a Run (the primitives that ask for samples, escalate, and judge) and the validated options as
+    # keywords, and returns the accepted typed result, or raises Squishling::InvalidOutputError (a
+    # DisagreementError when samples disagreed) or Squishling::LLMError, which squish_fallback handles.
+    class Base
+      Option = Data.define(:name, :type, :required)
+
+      # The kinds of parameter that can receive the run, call's first argument.
+      RUN_PARAMETERS = %i[req opt rest].freeze
+      KEYWORD_PARAMETERS = %i[key keyreq].freeze
+      REQUIRED_PARAMETERS = %i[req keyreq].freeze
+
+      class << self
+        # Declares an option: its name, one of OptionTypes::TYPES (or a callable that normalizes the value, see
+        # OptionTypes), and whether it must be given. A harness is declared with { type: :name, option_name: value },
+        # and the options are checked up front.
+        def option(name, type, required: false)
+          unless OptionTypes.type?(type)
+            raise ConfigurationError, "option #{name.inspect}: type must be one of #{OptionTypes::TYPES.join(', ')} " \
+                                      "or a callable taking (value, label, name), got #{type.inspect}"
+          end
+
+          raise ConfigurationError, "declare options on a subclass of Harness::Base, not on Base" if equal?(Base)
+          if name.to_sym == :type
+            raise ConfigurationError, "option :type is taken: it names the harness in a harness Hash"
+          end
+
+          own_options[name.to_sym] = Option.new(name: name.to_sym, type:, required: required ? true : false)
+        end
+
+        # The declared options by name, including those of the superclass.
+        def options
+          inherited = equal?(Base) ? {} : superclass.options
+          inherited.merge(own_options)
+        end
+
+        # The options as given in a harness Hash (without type:): checked and normalized, or ConfigurationError.
+        # An option given as nil counts as not given.
+        def normalize_options(given, label)
+          given = given.transform_keys(&:to_sym)
+          unknown = given.keys - options.keys
+          raise ConfigurationError, "#{label}: unknown harness option(s) #{unknown.join(', ')}" if unknown.any?
+
+          given = given.compact
+
+          missing = options.values.select { |option| option.required && !given.key?(option.name) }
+          if missing.any?
+            raise ConfigurationError, "#{label}: harness option(s) #{missing.map(&:name).join(', ')} required"
+          end
+
+          normalized = given.to_h do |name, value|
+            [name, OptionTypes.check(options.fetch(name).type, value, label, name)]
+          end
+          validate_options!(normalized, label)
+          normalized.freeze
+        end
+
+        # Override to check options against each other; raise ConfigurationError (with the label) to reject.
+        def validate_options!(_options, _label); end
+
+        # Rejects a class that can't run as the harness registered under name: not a Base subclass, no call,
+        # or a call that doesn't take the run and each declared option.
+        def check_registrable!(klass, name)
+          unless klass.is_a?(Class) && klass <= Base && klass != Base
+            raise ConfigurationError, "harness #{name.inspect} must be a subclass of Squishling::Harness::Base, " \
+                                      "got #{klass.inspect}"
+          end
+          if klass.instance_method(:call).owner == Base
+            raise ConfigurationError, "harness #{name.inspect} must define call(run, **options)"
+          end
+          unless klass.public_method_defined?(:call)
+            raise ConfigurationError, "harness #{name.inspect}: call must be public"
+          end
+
+          check_signature!(klass, name)
+        end
+
+        private
+
+        def own_options
+          @own_options ||= {}
+        end
+
+        def check_signature!(klass, name)
+          parameters = klass.instance_method(:call).parameters
+          unless RUN_PARAMETERS.include?(parameters.first&.first)
+            raise ConfigurationError, "harness #{name.inspect}: call must take the run as its first argument"
+          end
+          if parameters.count { |type, _| type == :req } > 1
+            raise ConfigurationError, "harness #{name.inspect}: call takes the run, then options as keywords only"
+          end
+          if klass.instance_method(:initialize).parameters.any? { |type, _| REQUIRED_PARAMETERS.include?(type) }
+            raise ConfigurationError, "harness #{name.inspect}: a harness is created without arguments, " \
+                                      "so initialize can't require any"
+          end
+
+          accepted = parameters.filter_map { |type, key| key if KEYWORD_PARAMETERS.include?(type) }
+          required = parameters.filter_map { |type, key| key if type == :keyreq }
+          # A ** takes every declared option, but not an option's absence.
+          missing = parameters.any? { |type, _| type == :keyrest } ? [] : klass.options.keys - accepted
+          undeclared = required - klass.options.keys
+          # An option that may be left out can't be a required keyword.
+          optional = required & klass.options.values.reject(&:required).map(&:name)
+          return if missing.empty? && undeclared.empty? && optional.empty?
+
+          problems = [("doesn't take #{missing.join(', ')}" if missing.any?),
+                      ("requires undeclared #{undeclared.join(', ')}" if undeclared.any?),
+                      ("requires optional #{optional.join(', ')} (give it a default)" if optional.any?)].compact
+          raise ConfigurationError, "harness #{name.inspect}: call #{problems.join(' and ')} " \
+                                    "(declare each option with option, and take it as a keyword)"
+        end
+      end
+
+      # The harness itself; subclasses define it.
+      def call(_run, **)
+        raise ConfigurationError, "#{self.class} must define call(run, **options)"
+      end
+    end
+  end
+end

data/lib/squishling/harness/escalation.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Squishling
+  class Harness
+    # The default: each attempt of the escalation, in order, until an output passes the schema and squish_validate.
+    class Escalation < Base
+      def self.normalize_options(given, label)
+        if given.transform_keys(&:to_sym).key?(:compare)
+          raise ConfigurationError, "#{label}: compare: only applies to the squishsum harnesses"
+        end
+
+        super
+      end
+
+      def call(run)
+        run.escalate
+      end
+    end
+
+    Registry.register_built_in(:escalation, Escalation)
+  end
+end

data/lib/squishling/harness/judged_squishsum.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Squishling
+  class Harness
+    # Like Squishling::Harness::Squishsum, but samples that disagree go to a judge, which picks one or rejects both.
+    # A judge is one step: a model name or a Hash with the escalation step keys (except priority:), plus
+    # type: (:chat, the default, or :judgment for a System One decision model through the adapter's judgments) and,
+    # for :judgment, min_confidence: (the probability the chosen candidate needs).
+    class JudgedSquishsum < Base
+      # Request keys the RubyLLM adapter's judgment protocols own (model and input are already reserved params).
+      JUDGMENT_RESERVED_PARAMS = %i[state questions].freeze
+      private_constant :JUDGMENT_RESERVED_PARAMS
+
+      class << self
+        # A judge as a normalized escalation step (see ModelPath), plus its type and min_confidence. label is the
+        # declaration site and name the option it came from. Internal.
+        def normalize_judge(judge, label, name = :judge)
+          judge_label = "#{label} #{name}"
+          return ModelPath.normalize_step(judge, judge_label).merge(type: :chat) unless judge.is_a?(Hash)
+
+          judge = judge.transform_keys(&:to_sym)
+          type = judge.fetch(:type, :chat)
+          type = type.to_sym if type.is_a?(String)
+          unless Judge::TYPES.include?(type)
+            raise ConfigurationError,
+              "#{judge_label}: type: must be one of #{Judge::TYPES.join(', ')}, got #{type.inspect}"
+          end
+
+          if type == :chat && judge.key?(:min_confidence)
+            raise ConfigurationError, "#{judge_label}: min_confidence: only applies to type: :judgment judges"
+          end
+
+          step = ModelPath.normalize_step(judge.except(*Judge::STEP_KEYS), judge_label).merge(type:)
+          ModelPath.single_step!(step, "#{judge_label}: a judge is one step, so it")
+          return step unless type == :judgment
+
+          Params.reject_reserved!(step[:params] || {}, JUDGMENT_RESERVED_PARAMS, judge_label)
+
+          step.merge(min_confidence: min_confidence(judge, judge_label))
+        end
+
+        # The judge's prompt: a non-blank String, or a Proc evaluated against the instance. Internal.
+        def normalize_instructions(value, label, name = :judge_instructions)
+          return value if value.is_a?(Proc)
+          return value.dup.freeze if value.is_a?(String) && !value.strip.empty?
+
+          raise ConfigurationError, "#{label}: #{name}: must be a non-blank String or a Proc"
+        end
+
+        private
+
+        def min_confidence(judge, label)
+          value = judge.fetch(:min_confidence, Judge::DEFAULT_MIN_CONFIDENCE)
+          return value.to_f if value.is_a?(Numeric) && value.between?(0, 1)
+
+          raise ConfigurationError, "#{label}: min_confidence: must be a number from 0 to 1, got #{value.inspect}"
+        end
+      end
+
+      option :compare, :proc
+      option :judge, ->(value, label, name) { normalize_judge(value, label, name) }
+      option :judge_instructions, ->(value, label, name) { normalize_instructions(value, label, name) }
+
+      def call(run, compare: nil, judge: nil, judge_instructions: nil)
+        stage = run.stages.first
+        # The judge is resolved before any sample is requested, so a missing one fails before costing anything.
+        judge = run.judge(judge:, instructions: judge_instructions)
+        first, second = run.samples([stage, stage])
+        return first if run.agree?(first, second, compare)
+
+        run.log("samples disagreed, asking the judge (#{judge.step.display_name})")
+        verdict = judge.verdict(first, second)
+        return { a: first, b: second }.fetch(verdict.choice) unless verdict.choice == :neither
+
+        models = [stage.first.model] * 2
+        raise DisagreementError.new(candidates: [first, second], verdict: :neither, reason: verdict.reason,
+          detail: verdict.detail, models: models + [judge.step.model])
+      end
+    end
+
+    Registry.register_built_in(:judged_squishsum, JudgedSquishsum)
+  end
+end

data/lib/squishling/harness/option_types.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Squishling
+  class Harness
+    # The types a harness option can declare (see Base.option). Each checks a given value and returns it
+    # normalized, or raises ConfigurationError, so a bad option fails before any request is made.
+    #   :step         a model name or a step Hash (model:, provider:, params:, attempts:, forward_rejected:,
+    #                 retry_temperature:)
+    #   :proc, :string, :symbol, :integer, :boolean
+    # A harness with a shape of its own declares the type as a callable instead, which gets the value, the
+    # declaration site, and the option's name, and returns the normalized value (or raises ConfigurationError).
+    module OptionTypes
+      TYPES = %i[step proc string symbol integer boolean].freeze
+
+      module_function
+
+      # Whether type is one a harness can declare: a built-in one above, or a callable that takes the value, the
+      # label, and the name.
+      def type?(type)
+        TYPES.include?(type) || (type.respond_to?(:call) && takes_three?(type))
+      end
+
+      # Whether the callable can be called with three positional arguments. One without a parameter list
+      # (an object with its own call) is trusted.
+      def takes_three?(callable)
+        return true if callable.is_a?(Proc) && !callable.lambda? # a proc drops the arguments it doesn't name
+
+        method = callable.is_a?(Proc) || callable.is_a?(Method) ? callable : callable.method(:call)
+        parameters = method.parameters.map(&:first)
+        return false if parameters.include?(:keyreq)
+        return true if parameters.include?(:rest)
+
+        3.between?(parameters.count(:req), parameters.count(:req) + parameters.count(:opt))
+      end
+
+      # The value as the type's normalized form. label is the declaration site, name the option.
+      def check(type, value, label, name)
+        return deep_freeze(type.call(value, label, name)) if type.respond_to?(:call)
+
+        send(:"check_#{type}", value, "#{label} #{name}", "#{label}: #{name}:")
+      end
+
+      def check_step(value, label, _where)
+        step = ModelPath.normalize_step(value, label)
+        ModelPath.single_step!(step, "#{label}: a harness step")
+        deep_freeze(step)
+      end
+
+      def check_proc(value, _label, where)
+        check_kind(value, Proc, "a Proc", where)
+      end
+
+      def check_string(value, _label, where)
+        return value.dup.freeze if value.is_a?(String)
+
+        raise ConfigurationError, "#{where} must be a String, got #{value.class}"
+      end
+
+      def check_symbol(value, _label, where)
+        return value.to_sym if value.is_a?(Symbol) || value.is_a?(String)
+
+        raise ConfigurationError, "#{where} must be a Symbol, got #{value.class}"
+      end
+
+      def check_integer(value, _label, where)
+        check_kind(value, Integer, "an Integer", where)
+      end
+
+      def check_boolean(value, _label, where)
+        return value if [true, false].include?(value)
+
+        raise ConfigurationError, "#{where} must be true or false, got #{value.inspect}"
+      end
+
+      # The value itself when it is a kind, else a ConfigurationError saying what was expected.
+      def check_kind(value, kind, description, where)
+        return value if value.is_a?(kind)
+
+        raise ConfigurationError, "#{where} must be #{description}, got #{value.class}"
+      end
+
+      # A frozen copy: normalized steps are shared by every call that uses the declaration, so none can change
+      # one, and the caller's own objects stay as they were.
+      def deep_freeze(value)
+        case value
+        when Hash then value.to_h { |key, item| [key, deep_freeze(item)] }.freeze
+        when Array then value.map { |item| deep_freeze(item) }.freeze
+        when String then value.dup.freeze
+        else value
+        end
+      end
+    end
+  end
+end

data/lib/squishling/harness/registry.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Squishling
+  class Harness
+    # The harness types by name: the built-ins, registered as they load, and your own (Squishling.register_harness).
+    module Registry
+      STORE = NamedRegistry.new("harness")
+      private_constant :STORE
+
+      class << self
+        def register_built_in(name, klass)
+          STORE.register_built_in(name, klass)
+        end
+
+        # Registers a harness class (or a block, which becomes the body of a Base subclass) under a name.
+        def register(name, klass = nil, &block)
+          raise ConfigurationError, "register_harness takes a class or a block, not both" if klass && block
+          raise ConfigurationError, "register_harness needs a Harness::Base subclass or a block" unless klass || block
+
+          name = STORE.checked_name(name)
+          klass = Class.new(Base, &block) if block
+          Base.check_registrable!(klass, name)
+          STORE.add(name, klass)
+        end
+
+        def fetch(name)
+          STORE.fetch(name)
+        end
+
+        def names
+          STORE.names
+        end
+
+        # Forgets the registered harnesses; the built-ins stay.
+        def reset!
+          STORE.reset!
+        end
+      end
+    end
+  end
+end

data/lib/squishling/harness/squishsum.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Squishling
+  class Harness
+    # Two samples on the escalation's first step, asked concurrently. Identical outputs (or ones compare: calls
+    # equal) are accepted; different ones raise DisagreementError.
+    class Squishsum < Base
+      option :compare, :proc
+
+      def call(run, compare: nil)
+        stage = run.stages.first
+        first, second = run.samples([stage, stage])
+        return first if run.agree?(first, second, compare)
+
+        run.log("samples disagreed")
+        raise DisagreementError.new(candidates: [first, second], models: [stage.first.model] * 2)
+      end
+    end
+
+    Registry.register_built_in(:squishsum, Squishsum)
+  end
+end

data/lib/squishling/harness.rb

@@ -5,24 +5,10 @@ module Squishling
   #   harness: :squishsum
   #   harness: { type: :judged_squishsum, judge: "claude-opus-5-5", compare: ->(a, b, **) { a.team == b.team } }
   # :escalation (the default) tries each attempt in order until an output passes. :squishsum asks the first
-  # escalation step twice, concurrently, and accepts the output only when both samples agree. :ensemble asks
-  # the first and second escalation steps once each instead (a cross-model check). :judged_squishsum and
-  # :judged_ensemble hand disagreeing samples to a judge that picks one or rejects both.
+  # escalation step twice, concurrently, and accepts the output only when both samples agree. :judged_squishsum
+  # hands disagreeing samples to a judge that picks one or rejects both. Your own types are registered with
+  # Squishling.register_harness (see Harness::Base).
   class Harness
-    TYPES = %i[escalation squishsum judged_squishsum ensemble judged_ensemble].freeze
-    ENSEMBLE_TYPES = %i[ensemble judged_ensemble].freeze
-    JUDGED_TYPES = %i[judged_squishsum judged_ensemble].freeze
-    KEYS = %i[type judge judge_instructions compare].freeze
-
-    # A judge is one step: a model name or a Hash with the escalation step keys (except order:), plus
-    # type: (:chat, the default, or :judgment for a System One decision model through RubyLLM.judge) and,
-    # for :judgment, min_confidence: (the probability the chosen candidate needs).
-    JUDGE_TYPES = %i[chat judgment].freeze
-    JUDGE_KEYS = %i[type min_confidence].freeze
-    # Request keys RubyLLM's judgment protocols own (model and input are already reserved params).
-    JUDGMENT_RESERVED_PARAMS = %i[state questions].freeze
-    DEFAULT_MIN_CONFIDENCE = 0.8
-
     DEFAULT_JUDGE_INSTRUCTIONS = <<~TEXT.strip
       You are an impartial judge. Two independent attempts at the same operation returned different outputs,
       candidate "a" and candidate "b". Both already match the required output format, so judge only their
@@ -32,122 +18,62 @@ module Squishling
       whether either is correct. Never combine them or invent a third answer.
     TEXT
 
-    attr_reader :type, :judge, :judge_instructions, :compare
+    # The type's validated options (frozen), and the Base subclass that runs it.
+    attr_reader :type, :klass, :options
 
     class << self
       def normalize(value, label)
         case value
-        when Symbol, String then new(type: type_for(value, label))
+        when Symbol, String then build(type_for(value, label), {}, label)
         when Hash then from_hash(value, label)
         else
-          raise ConfigurationError, "#{label}: harness must be one of #{TYPES.join(', ')} or a Hash with type:, " \
+          raise ConfigurationError, "#{label}: harness must be one of #{types.join(', ')} or a Hash with type:, " \
                                     "got #{value.inspect}"
         end
       end
 
+      # Every registered type: the built-ins, then your own in registration order.
+      def types
+        Registry.names
+      end
+
       private
 
       def from_hash(hash, label)
         hash = hash.transform_keys(&:to_sym)
-        unknown = hash.keys - KEYS
-        raise ConfigurationError, "#{label}: unknown harness option(s) #{unknown.join(', ')}" if unknown.any?
         raise ConfigurationError, "#{label}: a harness Hash needs type:" unless hash.key?(:type)
 
-        type = type_for(hash[:type], label)
-        check_options!(type, hash, label)
-        new(type:, judge: hash[:judge].nil? ? nil : normalize_judge(hash[:judge], label),
-          judge_instructions: hash[:judge_instructions], compare: hash[:compare])
+        build(type_for(hash[:type], label), hash.except(:type), label)
       end
 
-      def type_for(value, label)
-        type = value.to_s.to_sym if value.is_a?(Symbol) || value.is_a?(String)
-        return type if TYPES.include?(type)
-
-        raise ConfigurationError, "#{label}: unknown harness #{value.inspect} (use #{TYPES.join(', ')})"
+      def build(type, options, label)
+        klass = Registry.fetch(type)
+        new(type:, klass:, options: klass.normalize_options(options, label))
       end
 
-      def check_options!(type, hash, label)
-        judge_options = %i[judge judge_instructions].select { |key| hash.key?(key) }
-        if judge_options.any? && !JUDGED_TYPES.include?(type)
-          options = judge_options.map { |key| "#{key}:" }.join(", ")
-          raise ConfigurationError,
-            "#{label}: #{options} can only be used with the #{JUDGED_TYPES.join(' and ')} harnesses"
-        end
-        if hash.key?(:compare) && type == :escalation
-          raise ConfigurationError, "#{label}: compare: only applies to the sampling harnesses (squishsum and ensemble)"
-        end
-        unless hash[:compare].nil? || hash[:compare].is_a?(Proc)
-          raise ConfigurationError, "#{label}: compare: must be a Proc, got #{hash[:compare].class}"
-        end
-
-        instructions = hash[:judge_instructions]
-        return if instructions.nil? || instructions.is_a?(Proc)
-        return if instructions.is_a?(String) && !instructions.strip.empty?
-
-        raise ConfigurationError, "#{label}: judge_instructions: must be a non-blank String or a Proc"
-      end
-
-      # The judge as a normalized escalation step (see ModelPath), plus its type and min_confidence.
-      def normalize_judge(judge, label)
-        judge_label = "#{label} judge"
-        return ModelPath.normalize_step(judge, judge_label).merge(type: :chat) unless judge.is_a?(Hash)
-
-        judge = judge.transform_keys(&:to_sym)
-        raise ConfigurationError, "#{judge_label}: a judge is one step, so it takes no order:" if judge.key?(:order)
-
-        type = judge.fetch(:type, :chat)
-        type = type.to_sym if type.is_a?(String)
-        unless JUDGE_TYPES.include?(type)
-          raise ConfigurationError,
-            "#{judge_label}: type: must be one of #{JUDGE_TYPES.join(', ')}, got #{type.inspect}"
-        end
-
-        if type == :chat && judge.key?(:min_confidence)
-          raise ConfigurationError, "#{judge_label}: min_confidence: only applies to type: :judgment judges"
-        end
-
-        step = ModelPath.normalize_step(judge.except(*JUDGE_KEYS), judge_label).merge(type:)
-        return step unless type == :judgment
-
-        reserved = (step[:params] || {}).keys & JUDGMENT_RESERVED_PARAMS
-        if reserved.any?
-          raise ConfigurationError, "#{judge_label}: #{reserved.join(', ')} can't be set through params " \
-                                    "(controlled by Squishling/RubyLLM)"
-        end
-
-        step.merge(min_confidence: min_confidence(judge, judge_label))
-      end
-
-      def min_confidence(judge, label)
-        value = judge.fetch(:min_confidence, DEFAULT_MIN_CONFIDENCE)
-        return value.to_f if value.is_a?(Numeric) && value.between?(0, 1)
+      def type_for(value, label)
+        type = value.to_s.to_sym if value.is_a?(Symbol) || value.is_a?(String)
+        return type if Registry.fetch(type)
 
-        raise ConfigurationError, "#{label}: min_confidence: must be a number from 0 to 1, got #{value.inspect}"
+        raise ConfigurationError, "#{label}: unknown harness #{value.inspect} (use #{types.join(', ')})"
       end
     end
 
-    def initialize(type:, judge: nil, judge_instructions: nil, compare: nil)
+    def initialize(type:, klass: Registry.fetch(type), options: {})
       @type = type
-      @judge = judge
-      @judge_instructions = judge_instructions
-      @compare = compare
+      @klass = klass
+      @options = options.freeze
       freeze
     end
 
     DEFAULT = new(type: :escalation)
 
-    # Whether the call asks two samples (squishsum and ensemble harnesses) rather than walking the escalation.
     def squishsum?
-      type != :escalation
-    end
-
-    # Whether the two samples come from the first two escalation steps instead of both from the first.
-    def ensemble?
-      ENSEMBLE_TYPES.include?(type)
+      %i[squishsum judged_squishsum].include?(type)
     end
 
     def judged?
-      JUDGED_TYPES.include?(type)
+      type == :judged_squishsum
     end
   end
 end