squishling 0.3.0 → 0.5.0

data/lib/squishling/invoker.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Squishling
-  # The elastic path: sends a squished method's inputs through RubyLLM with structured output.
+  # The elastic path: sends a squished method's inputs through the configured adapter with structured output.
   class Invoker
     INPUT_NOTE = <<~NOTE
       The input is a JSON object. "arguments" holds the values passed to this operation; "context", when
@@ -15,192 +15,50 @@ module Squishling
     # role (:judge) skips squish_validate and names the judge in errors and logs.
     Prompt = Data.define(:instructions, :schema, :input, :role)
 
-    # One of a sampling harness's two samples, between rounds: the steps it may attempt, in order.
-    Sample = Struct.new(:name, :steps, :chat, :rejected, :result)
-
     def initialize(definition, receiver, inputs)
       @definition = definition
       @receiver = receiver
       @inputs = inputs
-      @client = LLMClient.new(definition.label)
-      @output_check = OutputCheck.new(definition, receiver, inputs)
     end
 
-    # Runs the harness (Definition#harness) over the escalation (Definition#escalation_path).
-    def call
-      purpose = @definition.purpose(@receiver)
-      schema = @definition.schema
-      raise ConfigurationError, "#{@definition.label} has no purpose" if purpose.nil? || purpose.empty?
-      raise ConfigurationError, "#{@definition.label} has no output_schema" unless schema
+    # Everything a Run needs of the call: the prompt, the payload it was built from, and the purpose text.
+    Built = Data.define(:prompt, :payload, :purpose)
 
+    # Runs the harness (Definition#harness) over a Run of the call: the harness decides how the escalation is
+    # used, and what it returns is checked against the output schema's type.
+    def call
+      built = build
       harness = @definition.harness
-      path = @definition.escalation_path
-      body = payload
-      prompt = Prompt.new(instructions: "#{purpose}\n\n#{INPUT_NOTE}", schema:,
-        input: JSON.generate(body), role: nil)
-      return escalate(path, prompt) unless harness.squishsum?
-
-      # The sample steps and the judge are resolved before any sample is requested, so a missing one fails
-      # before costing anything.
-      sample_steps, rest = sampling_plan(path, harness)
-      judge = harness.judged? && Judge.new(definition: @definition, receiver: @receiver, client: @client, harness:,
-        rest:, purpose:, payload: body, escalate: method(:escalate),
-        log_failure: method(:log_failure))
-      sample(sample_steps, prompt, harness, judge)
-    end
-
-    private
-
-    # Splits the escalation into runs of consecutive identical steps (a step's attempts). A squishsum samples
-    # the first run twice; an ensemble samples the first run, then the second. Returns each sample's steps and
-    # the rest of the escalation, which is where a judge looks for its default step.
-    def sampling_plan(path, harness)
-      runs = path.chunk_while { |step, following| step == following }.to_a
-      if harness.ensemble? && runs.size < 2
-        raise ConfigurationError, "#{@definition.label}: the #{harness.type} harness needs a second escalation " \
-                                  "step to sample (adjacent identical steps count as one step's attempts)"
-      end
-
-      return [runs.first(2), runs.drop(2).flatten(1)] if harness.ensemble?
-
-      [[runs.first, runs.first], runs.drop(1).flatten(1)]
-    end
-
-    # Makes each attempt of the path until an output passes the schema and the squish_validate check.
-    # Consecutive attempts on the same step continue the same conversation, so the model sees what it got wrong;
-    # a new step starts a fresh chat told about the last rejected output (unless the step sets
-    # forward_rejected: false).
-    def escalate(path, prompt)
-      chat = nil
-      rejected = nil # [raw content, errors] of the last invalid output
-      models = []
-
-      path.each.with_index(1) do |step, attempt|
-        models << step.model
-        last = attempt == path.size
-        if chat && step == path[attempt - 2]
-          message = retry_message(rejected.last)
-        else
-          chat = start_chat(step, prompt)
-          message = rejected && step.forward_rejected ? escalation_message(prompt.input, *rejected) : prompt.input
-        end
-
-        begin
-          response = @client.ask(chat, message, step)
-        rescue LLMError => e
-          squawk(prompt, path, attempt, nil, e)
-          raise if last
-
-          # The failed request may be half-recorded in this chat, so the next attempt starts a fresh one.
-          chat = nil
-          log_failure(prompt.role, attempt, path, e.message)
-          next
-        end
-
-        result, errors = @output_check.call(response.content, prompt)
-        if errors.empty?
-          squawk(prompt, path, attempt, response, nil)
-          return result
-        end
-
-        error = InvalidOutputError.new(errors:, raw: response.content, attempts: attempt, models: models.dup,
-          source: source(prompt.role))
-        squawk(prompt, path, attempt, response, error)
-        raise error if last
-
-        rejected = [response.content, errors]
-        log_failure(prompt.role, attempt, path, errors.join("; "))
-      end
-    end
-
-    # Two samples, each on its own steps and retried per their attempts the same way escalate retries.
-    # Each round sends the pending samples' requests concurrently; everything else runs on this thread.
-    def sample(sample_steps, prompt, harness, judge)
-      samples = %w[a b].zip(sample_steps).map { |name, steps| Sample.new(name, steps) }
-
-      1.upto(sample_steps.map(&:size).max) do |attempt|
-        pending = samples.reject(&:result)
-        break if pending.empty?
-
-        jobs = pending.map do |sample|
-          step = sample.steps[attempt - 1]
-          message = sample_message(sample, step, prompt)
-          -> { @client.ask(sample.chat, message, step) }
-        end
-        outcomes = LLMClient.concurrently(jobs)
-        # A setup mistake (or anything that isn't a failed request) ends the call, whichever sample hit it.
-        fatal = outcomes.map(&:last).find { |error| error && !error.is_a?(LLMError) }
-        raise fatal if fatal
-
-        pending.zip(outcomes).each { |sample, outcome| record(sample, outcome, prompt, attempt) }
-      end
-
-      settle(samples, harness, judge)
-    end
-
-    # The next message for a sample: a retry in its chat after invalid output, otherwise a fresh chat.
-    def sample_message(sample, step, prompt)
-      return retry_message(sample.rejected.last) if sample.chat
-
-      sample.chat = start_chat(step, prompt)
-      sample.rejected && step.forward_rejected ? escalation_message(prompt.input, *sample.rejected) : prompt.input
+      run = Run.new(definition: @definition, receiver: @receiver, inputs: @inputs, prompt: built.prompt,
+        payload: built.payload, purpose: built.purpose, harness:)
+      accepted(harness.klass.new.call(run, **harness.options), run, harness)
     end
 
-    def record(sample, (response, error), prompt, attempt)
-      steps = sample.steps
-      last = attempt == steps.size
-      role = "sample #{sample.name}"
-      if error
-        squawk(prompt, steps, attempt, nil, error)
-        raise error if last
-
-        sample.chat = nil
-        return log_failure(role, attempt, steps, error.message)
-      end
-
-      result, errors = @output_check.call(response.content, prompt)
-      if errors.empty?
-        squawk(prompt, steps, attempt, response, nil)
-        return sample.result = result
-      end
-
-      models = steps.first(attempt).map(&:model)
-      invalid = InvalidOutputError.new(errors:, raw: response.content, attempts: attempt, models:)
-      squawk(prompt, steps, attempt, response, invalid)
-      raise invalid if last
-
-      sample.rejected = [response.content, errors]
-      log_failure(role, attempt, steps, errors.join("; "))
+    # The first request the call would make (system prompt, output schema, user message), without making it.
+    def prompt
+      build.prompt
     end
 
-    # Agreeing samples are accepted; otherwise the judge, if any, picks one or the call fails.
-    def settle(samples, harness, judge)
-      first, second = samples.map(&:result)
-      return first if agree?(first, second, harness.compare)
-
-      models = samples.map { |sample| sample.steps.first.model }
-      unless judge
-        log_warning("samples disagreed")
-        raise DisagreementError.new(candidates: [first, second], models:)
-      end
+    private
 
-      log_warning("samples disagreed, asking the judge (#{judge.step.display_name})")
-      choice, reason, detail = judge.verdict(first, second)
-      return { a: first, b: second }.fetch(choice) unless choice == :neither
+    def build
+      purpose = @definition.purpose(@receiver)
+      schema = @definition.schema
+      raise ConfigurationError, "#{@definition.label} has no purpose" if purpose.nil? || purpose.empty?
+      raise ConfigurationError, "#{@definition.label} has no output_schema" unless schema
 
-      raise DisagreementError.new(candidates: [first, second], verdict: :neither, reason:, detail:,
-        models: models + [judge.step.model])
+      body = payload
+      prompt = Prompt.new(instructions: "#{purpose}\n\n#{INPUT_NOTE}", schema:, input: JSON.generate(body), role: nil)
+      Built.new(prompt:, payload: body, purpose:)
     end
 
-    # compare: is the developer's own code, so its exceptions propagate unwrapped.
-    def agree?(first, second, compare)
-      return Schema.jsonify(first.to_h) == Schema.jsonify(second.to_h) unless compare
+    # What a harness returns must be a result its run produced: validated output, typed like the deterministic
+    # path's. One built by hand, or a hash, would skip the schema and squish_validate.
+    def accepted(value, run, harness)
+      return value if run.produced?(value)
 
-      @receiver.instance_exec(first, second, **@inputs, &compare) ? true : false
-    end
-
-    def start_chat(step, prompt)
-      @client.chat(step, instructions: prompt.instructions, schema: prompt.schema.llm_schema)
+      raise ConfigurationError, "#{@definition.label}: the #{harness.type} harness must return a result from " \
+                                "its run (escalate, samples, or a judge's pick), got #{value.class}"
     end
 
     # Only the arguments, the declared squish_context names, and a squish! call's context leave the process.
@@ -233,68 +91,5 @@ module Squishling
 
       @receiver.instance_variable_get(:"@#{name}")
     end
-
-    def retry_message(errors)
-      "Your previous response was rejected:\n- #{errors.join("\n- ")}\nRespond again with corrected JSON only."
-    end
-
-    # The first message to a fresh chat after an earlier model's output was rejected. The rejected output is
-    # model-generated and may cross providers, so it is capped at MAX_FORWARDED_CHARS.
-    def escalation_message(input, raw, errors)
-      previous = rejected_text(raw)
-      previous = "(an empty response)" if previous.strip.empty? || raw.nil?
-      if previous.length > MAX_FORWARDED_CHARS
-        omitted = previous.length - MAX_FORWARDED_CHARS
-        previous = "#{previous[0, MAX_FORWARDED_CHARS]}... [truncated, #{omitted} more characters]"
-      end
-      "#{input}\n\nA previous attempt at this request returned:\n#{previous}\n" \
-        "It was rejected:\n- #{errors.join("\n- ")}\nRespond with corrected JSON only."
-    end
-
-    # The rejected output as text. Output that was rejected for being unrepresentable in JSON (Infinity, invalid
-    # UTF-8) can't be serialized or measured as is, so it is scrubbed or replaced rather than raising.
-    def rejected_text(raw)
-      return utf8(raw).scrub("?") if raw.is_a?(String)
-
-      JSON.generate(raw)
-    rescue JSON::JSONError
-      "(a response JSON can't represent)"
-    end
-
-    # A binary-tagged string counts every byte as valid, so it is read as UTF-8 before scrubbing.
-    def utf8(text)
-      text.encoding == Encoding::BINARY ? text.dup.force_encoding(Encoding::UTF_8) : text
-    end
-
-    # Runs the observability hook, if any, with this attempt's raw output (nil when the call itself failed),
-    # the error that ended it (nil when it was accepted), and what was asked.
-    def squawk(prompt, path, attempt, response, error)
-      hook = @definition.squawk
-      return unless hook
-
-      step = path[attempt - 1]
-      metadata = {
-        label: @definition.label, attempt:, attempts: path.size, final: attempt == path.size,
-        model: response&.model || step.model, provider: step.provider, params: step.params, input: prompt.input,
-        usage: response&.tokens&.to_h
-      }
-      Squawk.call(hook, output: response&.content, metadata:, error:)
-    end
-
-    def source(role)
-      role == :judge ? "Judge" : "LLM"
-    end
-
-    def log_failure(role, attempt, path, reason)
-      step = path[attempt - 1]
-      next_step = path[attempt]
-      action = next_step == step ? "retrying" : "escalating to #{next_step.display_name}"
-      who = role ? "#{role} attempt" : "attempt"
-      log_warning("#{who} #{attempt} of #{path.size} (#{step.display_name}) failed, #{action}: #{reason}")
-    end
-
-    def log_warning(message)
-      Squishling.config.logger&.warn("[Squishling] #{@definition.label} #{message}")
-    end
   end
 end

data/lib/squishling/judge.rb

@@ -1,12 +1,20 @@
 # frozen_string_literal: true
 
 module Squishling
-  # Picks between a judged harness's two disagreeing samples (judged_squishsum or judged_ensemble), or rejects
-  # both. The judge is the harness's judge: step, or else the escalation's next step after the ones the samples
-  # ran on. A chat judge answers with a strict verdict schema; a :judgment judge (a System One decision model such
-  # as Jev) answers one choice question through RubyLLM.judge, and its pick counts only at or above
+  # Picks between two disagreeing samples, or rejects both (see Run#judge). The judge is the given judge:
+  # step, or else the escalation's next stage after the first. A chat judge answers with a strict verdict
+  # schema; a :judgment judge (a System One decision model such as Jev) answers one choice question through
+  # the adapter's judgments (RubyLLM's, with the RubyLLM adapter), and its pick counts only at or above
   # min_confidence.
   class Judge
+    # The judge kinds: :chat (a chat model answering VERDICT_SCHEMA) and :judgment (a System One decision model
+    # through the adapter's judgments). A judge is an escalation step (see ModelPath) plus these keys:
+    # type: (default :chat)
+    # and, for :judgment, min_confidence: (the probability the chosen candidate needs).
+    TYPES = %i[chat judgment].freeze
+    STEP_KEYS = %i[type min_confidence].freeze
+    DEFAULT_MIN_CONFIDENCE = 0.8
+
     VERDICT_SCHEMA = {
       "type" => "object",
       "properties" => {
@@ -29,16 +37,29 @@ module Squishling
       neither: "Neither candidate is clearly correct"
     }.freeze
 
-    def initialize(definition:, receiver:, client:, harness:, rest:, purpose:, payload:, escalate:, log_failure:)
+    # What a judge decided: :a, :b, or :neither, with its reason. The detail is text Squishling wrote (nil for a
+    # chat judge, whose reason is model-written) and is safe for error messages.
+    Verdict = Data.define(:choice, :reason, :detail)
+
+    # judge: is a normalized judge step (see Harness::JudgedSquishsum) or nil for the escalation's next stage,
+    # instructions: a String, a Proc evaluated against the receiver, or nil for the default prompt.
+    def initialize(definition:, receiver:, client:, judge:, instructions:, path:, purpose:, payload:, escalate:,
+      log_failure:, guard:, budget_check:, harness_type:)
       @definition = definition
       @receiver = receiver
       @client = client
-      @harness = harness
+      @judge = judge
+      @instructions = instructions
+      check!
+      check_judgments!
+      @guard = guard
+      @budget_check = budget_check
+      @harness_type = harness_type
       @purpose = purpose
       @payload = payload
       @escalate = escalate
       @log_failure = log_failure
-      @steps = resolve_steps(rest)
+      @steps = resolve_steps(path)
     end
 
     # The judge's first attempt, for logs and DisagreementError#models.
@@ -46,31 +67,66 @@ module Squishling
       @steps.first
     end
 
-    # [:a | :b, reason, detail] for the chosen candidate, or [:neither, reason, detail]. The detail is text
-    # Squishling wrote (nil for a chat judge, whose reason is model-written) and is safe for error messages.
+    # The Verdict for the two candidates: :a or :b for the chosen one, or :neither.
     def verdict(first, second)
+      @guard.call
       candidates = { a: Schema.jsonify(first.to_h), b: Schema.jsonify(second.to_h) }
       judgment? ? judgment(candidates) : chat(candidates)
     end
 
     private
 
+    # What Run#judge was given. A harness declaration is checked earlier, with its own messages.
+    def check!
+      label = "#{@definition.label}: judge"
+      if @judge
+        type = @judge.fetch(:type)
+        unless TYPES.include?(type)
+          raise ConfigurationError, "#{label} type: must be one of #{TYPES.join(', ')}, got #{type.inspect}"
+        end
+
+        if type == :chat && @judge.key?(:min_confidence)
+          raise ConfigurationError, "#{label} min_confidence: only applies to type: :judgment judges"
+        end
+
+        minimum = @judge.fetch(:min_confidence, DEFAULT_MIN_CONFIDENCE)
+        unless minimum.is_a?(Numeric) && minimum.between?(0, 1)
+          raise ConfigurationError, "#{label} min_confidence: must be a number from 0 to 1, got #{minimum.inspect}"
+        end
+      end
+      return if @instructions.nil? || @instructions.is_a?(Proc) || non_blank_string?(@instructions)
+
+      raise ConfigurationError, "#{label} instructions must be a non-blank String or a Proc"
+    end
+
+    # Raised before any request, like the rest of the judge's checks.
+    def check_judgments!
+      return if !judgment? || @client.judgments?
+
+      raise ConfigurationError, "#{@definition.label}: judge type: :judgment needs an adapter that supports " \
+                                "judgments (the configured adapter doesn't); use type: :chat or the :ruby_llm adapter"
+    end
+
+    def non_blank_string?(value)
+      value.is_a?(String) && !value.strip.empty?
+    end
+
     def judgment?
-      @harness.judge&.fetch(:type) == :judgment
+      @judge&.fetch(:type) == :judgment
     end
 
     # A declared judge brings its own model and provider. A chat judge gets the method's generation params
     # under its own, like any escalation step; a judgment judge sends only its own params, as provider options.
-    def resolve_steps(rest)
-      if (judge = @harness.judge)
+    def resolve_steps(path)
+      if (judge = @judge)
         params = judge[:type] == :judgment ? {} : @definition.params
         return ModelPath.steps([judge], provider: nil, params:)
       end
 
+      rest = path.drop_while { |step| step == path.first }
       if rest.empty?
-        ordinal = @harness.ensemble? ? "third" : "second"
-        raise ConfigurationError, "#{@definition.label}: the #{@harness.type} harness needs a judge: or a " \
-                                  "#{ordinal} escalation step to judge with"
+        raise ConfigurationError, "#{@definition.label}: the #{@harness_type} harness needs a judge: or a second " \
+                                  "escalation step to judge with"
       end
 
       rest.take_while { |step| step == rest.first }
@@ -83,7 +139,7 @@ module Squishling
         schema: Schema.for(VERDICT_SCHEMA), input: JSON.generate(state(candidates)), role: :judge
       )
       result = @escalate.call(@steps, prompt)
-      [result[:verdict].to_sym, result[:reason], nil]
+      Verdict.new(choice: result[:verdict].to_sym, reason: result[:reason], detail: nil)
     end
 
     def judgment(candidates)
@@ -100,12 +156,12 @@ module Squishling
     # The threshold is checked on the exact probability; it's rounded only for the reason text.
     def interpret(choice, probabilities)
       probability = probabilities.to_h.find { |key, _| key.to_s == choice.to_s }&.last.to_f
-      minimum = @harness.judge[:min_confidence]
+      minimum = @judge.fetch(:min_confidence, DEFAULT_MIN_CONFIDENCE)
       shown = probability.round(3)
       picked = %i[a b].include?(choice)
       if picked && probability >= minimum
         text = "chosen with probability #{shown}"
-        return [choice, text, text]
+        return Verdict.new(choice:, reason: text, detail: text)
       end
 
       text =
@@ -113,13 +169,14 @@ module Squishling
         elsif picked then "the judge chose #{choice} with probability #{shown}, below min_confidence #{minimum}"
         else "the judge returned an unrecognized choice (probability #{shown})"
         end
-      [:neither, text, text]
+      Verdict.new(choice: :neither, reason: text, detail: text)
     end
 
     # A judgment always matches its questions, so only a failed request (LLMError) moves on to the next attempt.
     def judge_with_retries(input, questions)
       @steps.each.with_index(1) do |step, attempt|
-        return @client.judge(input, questions:, step:)
+        @budget_check.call(attempts: attempt - 1)
+        return @client.observed_judgment(input, questions:, step:, attempt:, attempts: @steps.size)
       rescue LLMError => e
         raise if attempt == @steps.size
 
@@ -132,7 +189,7 @@ module Squishling
     end
 
     def judge_instructions
-      value = @harness.judge_instructions || Harness::DEFAULT_JUDGE_INSTRUCTIONS
+      value = @instructions || Harness::DEFAULT_JUDGE_INSTRUCTIONS
       value = @receiver.instance_exec(&value) if value.is_a?(Proc)
       return value if value.is_a?(String) && !value.strip.empty?