squishling 0.2.0 → 0.4.0

data/lib/squishling/run.rb

@@ -0,0 +1,296 @@
+# frozen_string_literal: true
+
+module Squishling
+  # One squished call, as a harness sees it (see Harness::Base): the primitives that ask the model for outputs,
+  # and everything they do for free: model, provider, and params resolution, schema and squish_validate checks,
+  # retries in the same chat, escalation, squawk reports, and Squishling's own errors when attempts run out.
+  # Each returns typed results, the same type as the deterministic path.
+  #
+  # A stage is the attempts of one escalation step (a model with its attempts:), as an Array of
+  # ModelPath::Step. Primitives take stages, or anything that declares one (a model name or a step Hash).
+  #
+  # Primitives must be called from the thread that runs the harness. Only the provider requests inside them run
+  # on other threads; parsing, validation, and every callback (squish_validate, compare:, procs) stay on this one.
+  class Run
+    include Messages
+
+    private_constant :Messages
+
+    # One of #samples' outputs, between rounds.
+    Sample = Struct.new(:name, :stage, :chat, :rejected, :result)
+
+    SAMPLE_NAMES = ("a".."z").to_a.freeze
+    private_constant :Sample, :SAMPLE_NAMES
+
+    attr_reader :receiver, :inputs, :steps
+
+    def initialize(definition:, receiver:, inputs:, prompt:, payload:, purpose:, harness:)
+      @definition = definition
+      @receiver = receiver
+      @inputs = inputs
+      @prompt = prompt
+      @payload = payload
+      @purpose = purpose
+      @harness = harness
+      @steps = definition.escalation_path.freeze
+      @client = LLMClient.new(definition.label, purpose:, harness: harness.type, handed_off: definition.handed_off)
+      @output_check = OutputCheck.new(definition, receiver, inputs)
+      @owner = Thread.current
+      @produced = {}.compare_by_identity
+    end
+
+    # Whether value is a result one of this run's primitives returned (not one built by hand).
+    def produced?(value)
+      @produced.key?(value)
+    end
+
+    # The squished method, as "Class#method".
+    def label
+      @definition.label
+    end
+
+    # The escalation's steps grouped into stages: runs of identical consecutive steps, the attempts of one model.
+    def stages
+      @stages ||= steps.slice_when { |before, after| before != after }.map(&:freeze).freeze
+    end
+
+    # A stage from a model name or a step Hash (model:, provider:, params:, attempts:, forward_rejected:), the
+    # way a judge step is resolved: it keeps its own provider, and its params merge over the method's.
+    # A stage (an Array of steps) is returned as is, and a single step is a stage of one attempt.
+    def stage(declaration)
+      return checked_stage([declaration]) if declaration.is_a?(ModelPath::Step)
+
+      if declaration.is_a?(Array)
+        return checked_stage(declaration) if declaration.any? && declaration.all?(ModelPath::Step)
+
+        raise ConfigurationError, "#{label}: a stage is the steps of one model, got #{declaration.inspect}"
+      end
+
+      step = Harness::OptionTypes.check(:step, declaration, label, :"harness step")
+      ModelPath.steps([step], provider: nil, params: @definition.params).freeze
+    end
+
+    # Makes each attempt of the stages in order (the escalation's steps when none are given) until an output
+    # passes the schema and squish_validate. Consecutive attempts on the same step continue the same chat, so the
+    # model sees what it got wrong; a new step starts a fresh chat told about the last rejected output (unless
+    # the step sets forward_rejected: false). Raises InvalidOutputError, or LLMError, when the last attempt fails.
+    def escalate(*stages)
+      owner!
+      path = stages.empty? ? steps : stages.flat_map { |declared| stage(declared) }
+      escalate_with(path, @prompt)
+    end
+
+    # One sample per stage (the attempts of one model), each asked on its own chat, each retried per its stage's
+    # attempts like #escalate, with the requests of every round sent concurrently. Returns the outputs in order.
+    # Samples are named "a", "b", ... in squawk reports.
+    def samples(stages)
+      owner!
+      raise ConfigurationError, "#{label}: samples takes an Array of stages" unless stages.is_a?(Array)
+
+      stages = stages.map { |declared| stage(declared) }
+      raise ConfigurationError, "#{label}: samples needs at least one stage" if stages.empty?
+      if stages.size > SAMPLE_NAMES.size
+        raise ConfigurationError, "#{label}: samples takes at most #{SAMPLE_NAMES.size} stages"
+      end
+
+      if (mixed = stages.find { |sample_stage| sample_stage.uniq.size > 1 })
+        raise ConfigurationError, "#{label}: a sample is asked on one model, so its stage can't mix steps " \
+                                  "(got #{mixed.map(&:display_name).join(', ')})"
+      end
+
+      samples = stages.zip(SAMPLE_NAMES).map { |sample_stage, name| Sample.new(name, sample_stage) }
+      (1..stages.map(&:size).max).each do |attempt|
+        pending = samples.reject(&:result)
+        break if pending.empty?
+
+        round(pending, attempt)
+      end
+      samples.map(&:result)
+    end
+
+    # A judge that picks between two outputs (see Judge#verdict, which returns a Judge::Verdict), or rejects
+    # both. judge: is a model name or a Hash with model:, provider:, params:, and attempts: (as an escalation
+    # step, without priority:), plus type: :chat (default) or :judgment and, for :judgment, min_confidence:.
+    # Without one, the judge is the escalation's next stage after the first. instructions: is a String, or a
+    # Proc evaluated against the receiver, replacing Harness::DEFAULT_JUDGE_INSTRUCTIONS. The judge is resolved
+    # here, so a missing one raises ConfigurationError before any request is made.
+    def judge(judge: nil, instructions: nil)
+      owner!
+      judge = judge_step(judge) unless judge.nil?
+      Judge.new(definition: @definition, receiver: @receiver, client: @client, judge:, instructions:, path: steps,
+        purpose: @purpose, payload: @payload, escalate: method(:escalate_with), log_failure: method(:log_failure),
+        guard: method(:owner!), harness_type: @harness.type)
+    end
+
+    # Whether two outputs agree: compare (a Proc, evaluated against the receiver with the two results and the
+    # method's inputs, truthy for agreement) or, without one, whether they are exactly equal.
+    # compare is the developer's own code, so its exceptions propagate unwrapped.
+    def agree?(first, second, compare = nil)
+      owner!
+      return Schema.jsonify(first.to_h) == Schema.jsonify(second.to_h) unless compare
+
+      @receiver.instance_exec(first, second, **@inputs, &compare) ? true : false
+    end
+
+    # Logs a warning, "[Squishling] Class#method message", to Squishling.config.logger, if there is one.
+    def log(message)
+      Squishling.config.logger&.warn("[Squishling] #{label} #{message}")
+    end
+
+    private
+
+    # Steps made or changed by hand get the params checks a declared step does (no reserved keys).
+    def checked_stage(stage)
+      stage.each { |step| Params.normalize(step.params, "#{label} harness step params") }
+      stage.frozen? ? stage : stage.dup.freeze
+    end
+
+    # A judge declaration as the escalation step Judge takes: one step (no priority:), with its type.
+    def judge_step(declaration)
+      step = ModelPath.normalize_step(declaration, "#{label} judge", extras: Judge::STEP_KEYS)
+      ModelPath.single_step!(step, "#{label} judge: a judge is one step, so it")
+
+      type = step.fetch(:type, :chat)
+      type = :chat if type.nil?
+      step.merge(type: type.is_a?(String) ? type.to_sym : type)
+    end
+
+    def owner!
+      return if Thread.current.equal?(@owner)
+
+      raise ConfigurationError, "#{label}: a harness must call its run's primitives from the thread it runs on"
+    end
+
+    # Makes each attempt of the path until an output passes (see #escalate). The judge's prompt (role :judge)
+    # skips squish_validate and names the judge in errors and logs.
+    def escalate_with(path, prompt)
+      chat = nil
+      rejected = nil # [raw content, errors] of the last invalid output
+      models = []
+
+      path.each.with_index(1) do |step, attempt|
+        models << step.model
+        last = attempt == path.size
+        if chat && step == path[attempt - 2]
+          message = retry_message(rejected.last)
+        else
+          chat = start_chat(step, prompt)
+          message = rejected && step.forward_rejected ? escalation_message(prompt.input, *rejected) : prompt.input
+        end
+
+        begin
+          response = @client.ask(chat, message, step, attempt:, attempts: path.size, role: prompt.role)
+        rescue LLMError => e
+          squawk(prompt, path, attempt, nil, e)
+          raise if last
+
+          # The failed request may be half-recorded in this chat, so the next attempt starts a fresh one.
+          chat = nil
+          log_failure(prompt.role, attempt, path, e.message)
+          next
+        end
+
+        result, errors = @output_check.call(response.content, prompt)
+        if errors.empty?
+          squawk(prompt, path, attempt, response, nil)
+          return produce(result, prompt)
+        end
+
+        error = InvalidOutputError.new(errors:, raw: response.content, attempts: attempt, models: models.dup,
+          source: prompt.role == :judge ? "Judge" : "LLM")
+        squawk(prompt, path, attempt, response, error)
+        raise error if last
+
+        rejected = [response.content, errors]
+        log_failure(prompt.role, attempt, path, errors.join("; "))
+      end
+    end
+
+    # One round of #samples: the pending samples' requests, sent concurrently, then each outcome recorded.
+    def round(pending, attempt)
+      jobs = pending.map do |sample|
+        step = sample.stage[attempt - 1]
+        message = sample_message(sample, step)
+        chat = sample.chat
+        attempts = sample.stage.size
+        -> { @client.ask(chat, message, step, attempt:, attempts:, role: :sample, sample: sample.name) }
+      end
+      outcomes = LLMClient.concurrently(jobs)
+      # A setup mistake (or anything that isn't a failed request) ends the call, whichever sample hit it.
+      fatal = outcomes.map(&:last).find { |error| error && !error.is_a?(LLMError) }
+      raise fatal if fatal
+
+      pending.zip(outcomes).each { |sample, outcome| record(sample, outcome, attempt) }
+    end
+
+    # The next message for a sample: a retry in its chat after invalid output, otherwise a fresh chat.
+    def sample_message(sample, step)
+      return retry_message(sample.rejected.last) if sample.chat
+
+      sample.chat = start_chat(step, @prompt)
+      sample.rejected && step.forward_rejected ? escalation_message(@prompt.input, *sample.rejected) : @prompt.input
+    end
+
+    def record(sample, (response, error), attempt)
+      stage = sample.stage
+      last = attempt == stage.size
+      role = "sample #{sample.name}"
+      if error
+        squawk(@prompt, stage, attempt, nil, error, sample:)
+        raise error if last
+
+        sample.chat = nil
+        return log_failure(role, attempt, stage, error.message)
+      end
+
+      result, errors = @output_check.call(response.content, @prompt)
+      if errors.empty?
+        squawk(@prompt, stage, attempt, response, nil, sample:)
+        return sample.result = produce(result, @prompt)
+      end
+
+      invalid = InvalidOutputError.new(errors:, raw: response.content, attempts: attempt,
+        models: stage.first(attempt).map(&:model))
+      squawk(@prompt, stage, attempt, response, invalid, sample:)
+      raise invalid if last
+
+      sample.rejected = [response.content, errors]
+      log_failure(role, attempt, stage, errors.join("; "))
+    end
+
+    # Remembers a result of the method's own output (a judge's verdict isn't one), so a harness can only return
+    # what a primitive produced.
+    def produce(result, prompt)
+      @produced[result] = true unless prompt.role == :judge
+      result
+    end
+
+    def start_chat(step, prompt)
+      @client.chat(step, instructions: prompt.instructions, schema: prompt.schema.llm_schema)
+    end
+
+    # Runs the observability hook, if any, with this attempt's raw output (nil when the call itself failed),
+    # the error that ended it (nil when it was accepted), and what was asked. role is nil for escalation
+    # attempts, :sample (with the sample's name) or :judge.
+    def squawk(prompt, path, attempt, response, error, sample: nil)
+      hook = @definition.squawk
+      return unless hook
+
+      step = path[attempt - 1]
+      metadata = {
+        label:, attempt:, attempts: path.size, final: attempt == path.size,
+        model: response&.model || step.model, provider: step.provider, params: step.params, input: prompt.input,
+        usage: response&.tokens&.to_h, role: sample ? :sample : prompt.role, sample: sample&.name
+      }
+      Squawk.call(hook, output: response&.content, metadata:, error:)
+    end
+
+    def log_failure(role, attempt, path, reason)
+      step = path[attempt - 1]
+      next_step = path[attempt]
+      action = next_step == step ? "retrying" : "escalating to #{next_step.display_name}"
+      who = role ? "#{role} attempt" : "attempt"
+      log("#{who} #{attempt} of #{path.size} (#{step.display_name}) failed, #{action}: #{reason}")
+    end
+  end
+end

data/lib/squishling/source.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module Squishling
-  # Ruby source for append_instructions items: a class or module body, or a single method, rendered as a
+  # Ruby source for append_to_purpose items: a class or module body, or a single method, rendered as a
   # fenced code block. Parsed with Prism (a default gem since Ruby 3.3), loaded the first time it's needed.
   module Source
     # owner => { method name, or nil for the module itself => rendered source }. Weak keys, so classes replaced
@@ -30,7 +30,7 @@ module Squishling
       def module_source(mod)
         constant = constant_location(mod)
         sources = constant ? class_bodies(mod, constant) : method_defs(mod)
-        raise ConfigurationError, "append_instructions: source for #{mod.inspect} isn't available" if sources.empty?
+        raise ConfigurationError, "append_to_purpose: source for #{mod.inspect} isn't available" if sources.empty?
 
         [display_name(mod), sources.join("\n\n")]
       end
@@ -58,7 +58,7 @@ module Squishling
       def method_source(method)
         label = method_label(method)
         source = def_source(method)
-        raise ConfigurationError, "append_instructions: source for #{label} isn't available" unless source
+        raise ConfigurationError, "append_to_purpose: source for #{label} isn't available" unless source
 
         [label, source]
       end
@@ -84,7 +84,7 @@ module Squishling
       # A squished method resolves to Squishling's wrapper; show the implementation beneath it.
       def unwrap(method)
         Wrapper.implementation(method) or
-          raise ConfigurationError, "append_instructions: #{method.name} has no implementation to show"
+          raise ConfigurationError, "append_to_purpose: #{method.name} has no implementation to show"
       end
 
       # Named after the def that's shown, so an alias is labeled by its original name.
@@ -162,7 +162,7 @@ module Squishling
         text = File.read(file, encoding: Encoding::UTF_8).scrub
         [Prism.parse(text).value, text.lines]
       rescue SystemCallError => e
-        raise ConfigurationError, "append_instructions: can't read #{file} (#{e.class})"
+        raise ConfigurationError, "append_to_purpose: can't read #{file} (#{e.class})"
       end
 
       # Give the node's first line its source line's indentation (it may start mid-line, as in

data/lib/squishling/squawk.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Squishling
+  # The opt-in observability hook: a callable run after every LLM attempt with the attempt's raw output, so
+  # it can be sent somewhere (an error tracker, a tracing tool). It's the one sanctioned way for model output
+  # to leave the process; nothing runs unless one is configured.
+  module Squawk
+    KEYWORD_TYPES = %i[key keyreq].freeze
+    POSITIONAL_TYPES = %i[req opt].freeze
+    KEYWORD_CAPTURE_TYPES = %i[key keyreq keyrest].freeze
+    KEYWORDS = %i[output metadata error].freeze
+
+    module_function
+
+    # The hook itself, or nil/false to inherit/silence. Anything else must respond to call and take keywords
+    # only, which is checked here so a mismatch fails at setup, not inside the first attempt.
+    def validate(hook, where)
+      return hook if hook.nil? || hook == false
+
+      unless hook.respond_to?(:call)
+        raise ConfigurationError, "#{where} squawk must respond to call (or be false), got #{hook.inspect}"
+      end
+      if positional_only?(parameters(hook))
+        raise ConfigurationError,
+          "#{where} squawk is called with keywords (output:, metadata:, error:), not positional arguments"
+      end
+      unknown = parameters(hook).filter_map { |type, name| name if type == :keyreq && !KEYWORDS.include?(name) }
+      raise ConfigurationError, "#{where} squawk requires unknown keyword(s) #{unknown.join(', ')}" if unknown.any?
+
+      hook
+    end
+
+    # Calls the hook with the keywords it declares, or all of them when it takes **, so new metadata fields
+    # never break an existing lambda. Exceptions it raises are the user's own and propagate as-is.
+    def call(hook, **payload)
+      parameters = parameters(hook)
+      payload = payload.slice(*parameters.filter_map { |type, name| name if KEYWORD_TYPES.include?(type) }) \
+        unless parameters.any? { |type, _| type == :keyrest }
+      hook.call(**payload)
+    end
+
+    # Positional parameters, or a bare splat, would receive the keywords as a Hash (or not at all).
+    def positional_only?(parameters)
+      types = parameters.map(&:first)
+      types.intersect?(POSITIONAL_TYPES) || (types.include?(:rest) && !types.intersect?(KEYWORD_CAPTURE_TYPES))
+    end
+
+    def parameters(hook)
+      (hook.is_a?(Proc) || hook.is_a?(Method) ? hook : hook.method(:call)).parameters
+    end
+  end
+end

data/lib/squishling/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Squishling
-  VERSION = "0.2.0"
+  VERSION = "0.4.0"
 end

data/lib/squishling.rb

@@ -9,16 +9,33 @@ require_relative "squishling/version"
 require_relative "squishling/errors"
 require_relative "squishling/configuration"
 require_relative "squishling/params"
+require_relative "squishling/squawk"
+require_relative "squishling/observer"
+require_relative "squishling/observers/coolhand"
 require_relative "squishling/model_path"
+require_relative "squishling/rotation"
+require_relative "squishling/harness/option_types"
+require_relative "squishling/harness/registry"
+require_relative "squishling/harness/base"
+require_relative "squishling/harness/escalation"
+require_relative "squishling/harness/squishsum"
+require_relative "squishling/harness/judged_squishsum"
+require_relative "squishling/harness"
 require_relative "squishling/result"
 require_relative "squishling/schema"
 require_relative "squishling/source"
 require_relative "squishling/appendices"
 require_relative "squishling/definition"
+require_relative "squishling/llm_client"
+require_relative "squishling/output_check"
+require_relative "squishling/run/messages"
+require_relative "squishling/run"
 require_relative "squishling/invoker"
+require_relative "squishling/judge"
 require_relative "squishling/router"
 require_relative "squishling/wrapper"
 require_relative "squishling/class_methods"
+require_relative "squishling/collisions"
 
 # Include Squishling in a class to make it elastic: its squished methods either run their
 # Ruby implementation or send their inputs through an LLM and return schema-validated results.
@@ -36,10 +53,35 @@ module Squishling
       @config = Configuration.new
     end
 
+    # Forgets the registered harnesses (the built-ins stay), e.g. between specs. Kept apart from reset_config!
+    # because harnesses are registered once at boot, and a suite that resets its config between examples
+    # shouldn't lose them.
+    def reset_harnesses!
+      Harness::Registry.reset!
+    end
+
+    # Registers a custom harness under a name, usable anywhere a built-in type is (harness: :name, or
+    # harness: { type: :name, option: value }, on a class, a method, a squish! call, or config.default_harness).
+    # Pass a Harness::Base subclass, or a block that becomes the body of one (see Harness::Base). Registered
+    # globally, so do it before the classes that use it load (an initializer). Raises ConfigurationError for a
+    # name that is invalid or already taken, including the built-in :escalation, :squishsum, :judged_squishsum.
+    def register_harness(name, klass = nil, &)
+      Harness::Registry.register(name, klass, &)
+    end
+
+    # The names of the harnesses that can be declared: the built-ins, then those registered.
+    def harnesses
+      Harness.types
+    end
+
     def included(base)
       raise ConfigurationError, "Squishling can only be included in a class" unless base.is_a?(Class)
 
+      # A redundant include in a subclass overrides nothing new: the superclass's own methods already won.
+      Collisions.check!(base) unless base.superclass&.include?(Squishling)
       base.extend(ClassMethods)
+      # `result` is a convenience alias for squishling_result; a class that already has a `result` keeps it.
+      base.include(ResultAlias) unless base.method_defined?(:result) || base.private_method_defined?(:result)
       base.send(:squishling_install_wrapper)
     end
   end
@@ -53,16 +95,23 @@ module Squishling
     frame.definition.build_result(attrs || kwargs, squished: false)
   end
 
-  alias_method :result, :squishling_result
+  # Included separately, and only when the class has no `result` of its own, so `result` never shadows an
+  # inherited one. A module (not an alias on the class) keeps it out of the class's own source.
+  module ResultAlias
+    def result(...)
+      squishling_result(...)
+    end
+  end
 
   # Hand the squished method currently executing to the LLM, e.g. from a `rescue` when the Ruby path can't
   # handle this input. Returns the typed result (squished? true, or false when the declared fallback supplied
-  # it); return it from the method. The overrides apply to this call only: append_instructions adds to (or,
+  # it); return it from the method. The overrides apply to this call only: append_to_purpose adds to (or,
   # with false, replaces) the declared sections, context is sent alongside the declared squish_context,
-  # model/escalation (one or the other)/provider/instructions replace the declared ones, and params merge key
-  # by key over them.
-  def squish!(append_instructions: nil, context: nil, instructions: nil, model: nil, escalation: nil, provider: nil,
-    params: nil)
-    Router.hand_off(self, { append_instructions:, context:, instructions:, model:, escalation:, provider:, params: })
+  # model/escalation (one or the other)/provider/purpose/harness replace the declared ones, and params merge
+  # key by key over them.
+  def squish!(append_to_purpose: nil, context: nil, purpose: nil, model: nil, escalation: nil, provider: nil,
+    params: nil, harness: nil)
+    Router.hand_off(self, { append_to_purpose:, context:, purpose:, model:, escalation:, provider:, params:,
+                            harness: })
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: squishling
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Michael Carroll
@@ -29,14 +29,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: schematist
   requirement: !ruby/object:Gem::Requirement
@@ -71,22 +71,44 @@ files:
 - Rakefile
 - SECURITY.md
 - docs/configuration.md
+- docs/coolhand.md
 - docs/failures.md
+- docs/harnesses.md
+- docs/measuring-tokens.md
+- docs/naming.md
+- docs/observers.md
 - docs/routing.md
 - docs/schemas.md
 - lib/squishling.rb
 - lib/squishling/appendices.rb
 - lib/squishling/class_methods.rb
+- lib/squishling/collisions.rb
 - lib/squishling/configuration.rb
 - lib/squishling/definition.rb
 - lib/squishling/errors.rb
+- lib/squishling/harness.rb
+- lib/squishling/harness/base.rb
+- lib/squishling/harness/escalation.rb
+- lib/squishling/harness/judged_squishsum.rb
+- lib/squishling/harness/option_types.rb
+- lib/squishling/harness/registry.rb
+- lib/squishling/harness/squishsum.rb
 - lib/squishling/invoker.rb
+- lib/squishling/judge.rb
+- lib/squishling/llm_client.rb
 - lib/squishling/model_path.rb
+- lib/squishling/observer.rb
+- lib/squishling/observers/coolhand.rb
+- lib/squishling/output_check.rb
 - lib/squishling/params.rb
 - lib/squishling/result.rb
+- lib/squishling/rotation.rb
 - lib/squishling/router.rb
+- lib/squishling/run.rb
+- lib/squishling/run/messages.rb
 - lib/squishling/schema.rb
 - lib/squishling/source.rb
+- lib/squishling/squawk.rb
 - lib/squishling/version.rb
 - lib/squishling/wrapper.rb
 homepage: https://github.com/Coolhand-Labs/squishling