squishling 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +133 -1
  3. data/README.md +63 -15
  4. data/docs/adapters.md +189 -0
  5. data/docs/configuration.md +89 -24
  6. data/docs/coolhand.md +66 -0
  7. data/docs/failures.md +129 -20
  8. data/docs/harnesses.md +186 -48
  9. data/docs/measuring-tokens.md +36 -42
  10. data/docs/observers.md +80 -0
  11. data/docs/routing.md +7 -2
  12. data/docs/schemas.md +33 -6
  13. data/docs/testing.md +183 -0
  14. data/docs/verifying.md +84 -0
  15. data/lib/squishling/adapter/error_body.rb +47 -0
  16. data/lib/squishling/adapter.rb +275 -0
  17. data/lib/squishling/adapters/openai_compatible.rb +304 -0
  18. data/lib/squishling/adapters/openai_options.rb +72 -0
  19. data/lib/squishling/adapters/ruby_llm.rb +193 -0
  20. data/lib/squishling/budget.rb +71 -0
  21. data/lib/squishling/class_methods.rb +43 -7
  22. data/lib/squishling/class_registry.rb +63 -0
  23. data/lib/squishling/configuration.rb +72 -8
  24. data/lib/squishling/definition.rb +95 -17
  25. data/lib/squishling/errors.rb +97 -12
  26. data/lib/squishling/harness/base.rb +136 -0
  27. data/lib/squishling/harness/escalation.rb +22 -0
  28. data/lib/squishling/harness/judged_squishsum.rb +83 -0
  29. data/lib/squishling/harness/option_types.rb +94 -0
  30. data/lib/squishling/harness/registry.rb +41 -0
  31. data/lib/squishling/harness/squishsum.rb +22 -0
  32. data/lib/squishling/harness.rb +25 -99
  33. data/lib/squishling/invoker.rb +28 -233
  34. data/lib/squishling/judge.rb +78 -21
  35. data/lib/squishling/llm_client.rb +73 -109
  36. data/lib/squishling/model_path.rb +117 -27
  37. data/lib/squishling/named_registry.rb +64 -0
  38. data/lib/squishling/observer.rb +109 -0
  39. data/lib/squishling/observers/coolhand.rb +86 -0
  40. data/lib/squishling/output_check.rb +13 -9
  41. data/lib/squishling/params.rb +15 -27
  42. data/lib/squishling/railtie.rb +10 -0
  43. data/lib/squishling/result.rb +37 -16
  44. data/lib/squishling/rotation.rb +31 -0
  45. data/lib/squishling/rspec.rb +18 -0
  46. data/lib/squishling/run/messages.rb +79 -0
  47. data/lib/squishling/run/rejections.rb +43 -0
  48. data/lib/squishling/run.rb +320 -0
  49. data/lib/squishling/schema.rb +10 -9
  50. data/lib/squishling/squawk.rb +33 -3
  51. data/lib/squishling/tasks/squishling.rake +18 -0
  52. data/lib/squishling/testing/adapter.rb +194 -0
  53. data/lib/squishling/testing/checks.rb +39 -0
  54. data/lib/squishling/testing/direct.rb +41 -0
  55. data/lib/squishling/testing/errors.rb +18 -0
  56. data/lib/squishling/testing/fixtures.rb +206 -0
  57. data/lib/squishling/testing/helpers.rb +71 -0
  58. data/lib/squishling/testing/minitest.rb +32 -0
  59. data/lib/squishling/testing/observer.rb +31 -0
  60. data/lib/squishling/testing/reply.rb +60 -0
  61. data/lib/squishling/testing/request.rb +75 -0
  62. data/lib/squishling/testing/rspec.rb +30 -0
  63. data/lib/squishling/testing/script.rb +189 -0
  64. data/lib/squishling/testing/state.rb +73 -0
  65. data/lib/squishling/testing/stubs.rb +112 -0
  66. data/lib/squishling/testing.rb +126 -0
  67. data/lib/squishling/verification.rb +39 -0
  68. data/lib/squishling/verifier.rb +179 -0
  69. data/lib/squishling/version.rb +1 -1
  70. data/lib/squishling.rb +89 -4
  71. metadata +53 -21
@@ -1,25 +1,41 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Squishling
4
- # RubyLLM requests for one squished method, with RubyLLM's errors mapped onto Squishling's taxonomy.
4
+ # The requests for one squished method, made through the configured adapter (see Adapter). Squishling owns the
5
+ # conversation: each Conversation holds the system prompt, the output schema, and the messages so far, and every
6
+ # request sends all of it to the adapter.
5
7
  class LLMClient
6
- # Where RubyLLM's own code lives, to tell its ArgumentErrors from the developer's (see #judge).
7
- RUBY_LLM_LIB = File.dirname(RubyLLM.method(:judge).source_location.first)
8
- private_constant :RUBY_LLM_LIB
8
+ # One chat on one escalation step. messages are { role:, content: } Hashes, grown by #ask.
9
+ Conversation = Struct.new(:step, :instructions, :schema, :messages)
10
+
11
+ # purpose, harness (the harness type), and handed_off (squish! sent the call here) are only told to the observer.
12
+ # budget (see Budget) caps the tokens the requests of this client may use between them.
13
+ def initialize(label, purpose:, harness:, handed_off:, budget: {})
14
+ @label = label
15
+ @purpose = purpose
16
+ @harness = harness
17
+ @handed_off = handed_off
18
+ @adapter = Squishling.config.resolved_adapter
19
+ @tally = Budget::Tally.new(budget)
20
+ end
21
+
22
+ # The budget limit this client's requests have reached ({ limit:, max:, used: }), or nil. Checked before each
23
+ # request, so one already in flight is never cut short.
24
+ def exhausted_budget
25
+ @tally.exhausted
26
+ end
9
27
 
10
28
  # Runs each job on its own thread and returns, in order, each job's [value, exception]. Only provider
11
29
  # requests run here: parsing, validation, and the developer's callbacks stay on the caller's thread. Fiber
12
- # storage (RubyLLM's usage owner) is inherited by the threads, and RubyLLM's instrumentation context is
13
- # carried over the way RubyLLM's own concurrent tool calls do it. Unlike those, the threads don't enter the
14
- # Rails executor: the caller already holds it, and a second share can deadlock against a pending code reload.
15
- # If the caller is interrupted while waiting (a timeout, Thread#raise), the requests still running are
16
- # stopped rather than left running unobserved.
17
- def self.concurrently(jobs)
30
+ # storage is inherited by the threads, and the adapter carries over whatever request context it keeps (see
31
+ # Adapter::Base#capture_context). If the caller is interrupted while waiting (a timeout, Thread#raise), the
32
+ # requests still running are stopped rather than left running unobserved.
33
+ def concurrently(jobs)
18
34
  return jobs.map { |job| capture(job) } if jobs.size < 2
19
35
 
20
- context = instrumentation_context
36
+ context = @adapter.capture_context if @adapter.respond_to?(:capture_context)
21
37
  threads = jobs.map do |job|
22
- thread = Thread.new { with_instrumentation_context(context) { capture(job) } }
38
+ thread = Thread.new { with_context(context) { capture(job) } }
23
39
  thread.report_on_exception = false
24
40
  thread
25
41
  end
@@ -28,123 +44,71 @@ module Squishling
28
44
  threads&.each { |thread| thread.kill.join if thread.alive? }
29
45
  end
30
46
 
31
- def self.capture(job)
32
- [job.call, nil]
33
- rescue Exception => e # rubocop:disable Lint/RescueException -- re-raised on the caller's thread
34
- [nil, e]
35
- end
36
-
37
- def self.instrumentation_context
38
- instrumentation = defined?(RubyLLM::Support::Instrumentation) && RubyLLM::Support::Instrumentation
39
- return unless instrumentation.respond_to?(:current_workflow) && instrumentation.respond_to?(:capture_context)
40
-
41
- [instrumentation.current_workflow, instrumentation.capture_context]
42
- end
43
-
44
- def self.with_instrumentation_context(context, &)
45
- return yield unless context
46
-
47
- instrumentation = RubyLLM::Support::Instrumentation
48
- instrumentation.with_workflow(context.first) { instrumentation.with_context(context.last, &) }
47
+ # A fresh conversation on one escalation step, with the system prompt and the strict output schema.
48
+ def chat(step, instructions:, schema:)
49
+ Conversation.new(step, instructions, schema, [])
49
50
  end
50
51
 
51
- private_class_method :capture, :instrumentation_context, :with_instrumentation_context
52
-
53
- def initialize(label)
54
- @label = label
52
+ # One request in the conversation: the message goes in, and the adapter's answer is kept in it so the next
53
+ # message continues from there. A request that fails leaves the conversation as it was.
54
+ # The adapter raises ConfigurationError for setup mistakes (never retried) and LLMError for failures worth another
55
+ # attempt, which moves on to the next attempt of the escalation (or propagates from the last one).
56
+ # The request runs inside the configured observer (see Observer), on this thread: for samples, the worker's.
57
+ # role is nil for an escalation attempt, :sample (with its name) or :judge; attempt counts in attempts.
58
+ # reask is whether this message is a retry in the same chat (see ModelPath::Step#request_params).
59
+ def ask(chat, message, step, attempt:, attempts:, role: nil, sample: nil, reask: false)
60
+ call = call_info(step, :chat, role:, sample:, attempt:, attempts:)
61
+ Observer.around(call) { complete(chat, message, step, reask) }
55
62
  end
56
63
 
57
- # A fresh chat on one escalation step, with the system prompt, the strict output schema, and its params.
58
- def chat(step, instructions:, schema:)
59
- chat = build_chat(step)
60
- chat.with_instructions(instructions)
61
- chat.with_schema(schema)
62
- apply_params(chat, step.params)
63
- chat
64
+ # Whether the adapter can make System One judgments (judges with type: :judgment).
65
+ def judgments?
66
+ @adapter.respond_to?(:judgments?) && @adapter.respond_to?(:judge) && @adapter.judgments?
64
67
  end
65
68
 
66
- # Transient HTTP failures are already retried by RubyLLM (RubyLLM.config.max_retries); anything
67
- # that still fails is surfaced as an LLMError, which moves on to the next attempt of the
68
- # escalation (or propagates from the last one). A 400 means the request we built is invalid (an
69
- # unsupported param, a schema the provider rejects), so it's a setup mistake: escalating or a
70
- # fallback would otherwise hide it on every call.
71
- def ask(chat, message, step)
72
- request(step) { chat.ask(message) }
69
+ # A System One judgment on one step, through the adapter.
70
+ # Judge#check_judgments! has already refused an adapter that can't.
71
+ def judge(input, questions:, step:)
72
+ @tally.add(@adapter.judge(input, questions:, step:, label: @label))
73
73
  end
74
74
 
75
- # A System One judgment (RubyLLM.judge) on one step. The step's params are sent as provider options.
76
- # RubyLLM raises ArgumentError for a judgment it can't build (e.g. a model on a provider that takes none, or
77
- # a provider option its protocol reserves), and a bare RubyLLM::Error without an HTTP response for a provider
78
- # that doesn't support judgments at all. An ArgumentError from anywhere else (an instrumentation subscriber
79
- # runs inside the call) is the developer's own and propagates as-is.
80
- def judge(input, questions:, step:)
81
- request(step) do
82
- RubyLLM.judge(input, questions:, provider_options: step.params, **model_options(step))
83
- rescue RubyLLM::ModelNotFoundError => e
84
- raise ConfigurationError, "#{@label}: #{e.message}"
85
- rescue ArgumentError => e
86
- raise unless raised_by_ruby_llm?(e)
87
-
88
- raise ConfigurationError, "#{@label}: #{e.message}"
89
- rescue RubyLLM::Error => e
90
- raise unless e.instance_of?(RubyLLM::Error) && e.response.nil?
91
-
92
- raise ConfigurationError, "#{@label}: #{e.message}"
93
- end
75
+ # #judge inside the configured observer (see Observer), attempt counting in attempts.
76
+ def observed_judgment(input, questions:, step:, attempt:, attempts:)
77
+ call = call_info(step, :judgment, role: :judge, attempt:, attempts:)
78
+ Observer.around(call) { judge(input, questions:, step:) }
94
79
  end
95
80
 
96
81
  private
97
82
 
98
- def raised_by_ruby_llm?(error)
99
- error.backtrace&.first&.start_with?("#{RUBY_LLM_LIB}/") || false
100
- end
101
-
102
- def build_chat(step)
103
- RubyLLM.chat(**model_options(step))
104
- rescue RubyLLM::ModelNotFoundError, RubyLLM::ConfigurationError => e
105
- raise ConfigurationError, "#{@label}: #{e.message}"
83
+ def capture(job)
84
+ [job.call, nil]
85
+ rescue Exception => e # rubocop:disable Lint/RescueException -- re-raised on the caller's thread
86
+ [nil, e]
106
87
  end
107
88
 
108
- # RubyLLM validates some settings locally (e.g. an impossible thinking budget for the model)
109
- # and raises ArgumentError before any request is sent.
110
- def apply_params(chat, params)
111
- Params.apply(chat, params)
112
- rescue ArgumentError => e
113
- raise ConfigurationError, "#{@label}: invalid params #{params.inspect} (#{e.message})"
114
- end
89
+ def complete(chat, message, step, reask)
90
+ messages = chat.messages + [{ role: :user, content: message }]
91
+ request = Adapter::Request.new(label: @label, model: step.model, provider: step.provider,
92
+ instructions: chat.instructions, messages:, schema: chat.schema, params: step.request_params(reask))
93
+ response = @adapter.complete(request)
94
+ unless response.is_a?(Adapter::Response)
95
+ raise ConfigurationError, "#{@label}: the #{@adapter.class} adapter must return a " \
96
+ "Squishling::Adapter::Response from #complete, got #{response.class}"
97
+ end
115
98
 
116
- def request(step)
117
- yield
118
- rescue RubyLLM::ConfigurationError, RubyLLM::UnauthorizedError, RubyLLM::ForbiddenError => e
119
- raise ConfigurationError, "#{@label}: #{e.class}: #{e.message}"
120
- rescue RubyLLM::BadRequestError => e
121
- raise ConfigurationError,
122
- "#{@label}: the provider rejected the request (#{e.message})#{params_hint(step.params)}"
123
- rescue RubyLLM::Error, Faraday::Error => e
124
- raise LLMError, "#{@label}: #{e.class}: #{e.message}"
99
+ chat.messages = messages + [{ role: :assistant, content: response.text }]
100
+ @tally.add(response)
125
101
  end
126
102
 
127
- # Models missing from RubyLLM's registry (e.g. newly released ones) are only usable when a
128
- # provider is named, so RubyLLM is told to assume they exist.
129
- def model_options(step)
130
- model = step.model
131
- provider = step.provider
132
- options = { model:, provider: }.compact
133
- options[:assume_model_exists] = true if model && provider && !known_model?(model, provider)
134
- options
135
- end
103
+ def with_context(context, &)
104
+ return yield unless @adapter.respond_to?(:with_context)
136
105
 
137
- def known_model?(model, provider)
138
- RubyLLM.models.find(model, provider:)
139
- true
140
- rescue RubyLLM::ModelNotFoundError
141
- false
106
+ @adapter.with_context(context, &)
142
107
  end
143
108
 
144
- def params_hint(params)
145
- return "" if params.empty?
146
-
147
- ". Check params #{params.inspect}; reasoning models often reject sampling params such as temperature and top_p"
109
+ def call_info(step, request, attempt:, attempts:, role: nil, sample: nil)
110
+ Observer::Call.new(label: @label, purpose: @purpose, harness: @harness, handed_off: @handed_off, role:, sample:,
111
+ attempt:, attempts:, model: step.model, provider: step.provider, request:)
148
112
  end
149
113
  end
150
114
  end
@@ -8,20 +8,35 @@ module Squishling
8
8
  # "claude-sonnet-5-5",
9
9
  # { model: "claude-opus-5-5", params: { thinking: { effort: :high } } }
10
10
  # ]
11
- # attempts: (default 1) retries a step; order: (all steps or none, unique, lowest first) makes the order
12
- # explicit instead of positional. forward_rejected: false starts a step from the original input alone, without
13
- # the previous step's rejected output.
11
+ # attempts: (default 1) retries a step; priority: (all steps or none, lowest first) makes the order explicit
12
+ # instead of positional. Steps sharing a priority are one tier: each call rotates which of them goes first
13
+ # (see Rotation), and the others follow before the next tier, unless a step in the tier sets rotate: false.
14
+ # order: is the deprecated name of priority:. forward_rejected: false starts a step from the original input
15
+ # alone, without the previous step's rejected output. include_when: (a Proc, called like squish_when with the
16
+ # method's inputs as keywords, on the instance) drops the step from any call where it is falsy.
17
+ # retry_temperature: (a Numeric) is the temperature for the step's same-chat retries after the first attempt.
14
18
  module ModelPath
15
- # One attempt: the model, its provider, the fully resolved generation params, and whether the previous
16
- # step's rejected output is shown to it.
17
- Step = Data.define(:model, :provider, :params, :forward_rejected) do
18
- # The model for logs and messages (nil means RubyLLM's default model).
19
+ # One attempt: the model, its provider, the fully resolved generation params, whether the previous
20
+ # step's rejected output is shown to it, and the temperature its retries in the same chat use, if any.
21
+ Step = Data.define(:model, :provider, :params, :forward_rejected, :retry_temperature) do
22
+ def initialize(model:, provider:, params:, forward_rejected:, retry_temperature: nil)
23
+ super
24
+ end
25
+
26
+ # The params for a request: on a retry in the same chat (reask), the step's retry_temperature replaces the
27
+ # temperature, if it sets one.
28
+ def request_params(reask)
29
+ reask && retry_temperature ? Params.resolve(params, temperature: retry_temperature) : params
30
+ end
31
+
32
+ # The model for logs and messages (nil means the adapter's default model).
19
33
  def display_name
20
- model || "RubyLLM default model"
34
+ model || "the adapter's default model"
21
35
  end
22
36
  end
23
37
 
24
- STEP_KEYS = %i[model provider params attempts order forward_rejected].freeze
38
+ STEP_KEYS = %i[model provider params attempts priority order rotate forward_rejected include_when
39
+ retry_temperature].freeze
25
40
 
26
41
  module_function
27
42
 
@@ -57,43 +72,75 @@ module Squishling
57
72
  end
58
73
 
59
74
  # One Step per attempt. The level's provider applies to steps that don't name their own, and each
60
- # step's params are merged over the level-resolved params.
61
- def steps(path, provider:, params:)
62
- path.flat_map do |step|
75
+ # step's params are merged over the level-resolved params. turn is this call's place in the method's rotation
76
+ # (see Rotation): each tier of tied priorities starts turn steps further along.
77
+ def steps(path, provider:, params:, turn: 0)
78
+ rotated(path, turn).flat_map do |step|
63
79
  attempt = Step.new(model: step[:model], provider: step[:provider] || provider,
64
- params: Params.resolve(params, step[:params]), forward_rejected: step[:forward_rejected])
80
+ params: Params.resolve(params, step[:params]), forward_rejected: step[:forward_rejected],
81
+ retry_temperature: step[:retry_temperature])
65
82
  [attempt] * step[:attempts]
66
83
  end
67
84
  end
68
85
 
86
+ # The steps this call may use: those without an include_when:, and those whose include_when: is truthy for the
87
+ # call's inputs (evaluated against the receiver, like squish_when). Priority order is kept.
88
+ def eligible(path, receiver, inputs, label)
89
+ kept = path.select do |step|
90
+ gate = step[:include_when]
91
+ gate.nil? || receiver.instance_exec(**inputs, &gate)
92
+ end
93
+ raise ConfigurationError, "#{label}: include_when: excluded every escalation step" if kept.empty?
94
+
95
+ kept
96
+ end
97
+
98
+ # Whether any tier of the path has several models to rotate between.
99
+ def rotating?(path)
100
+ tiers(path).any? { |tier| rotates?(tier) }
101
+ end
102
+
103
+ # Steps sorted by priority, lowest first, ties kept in the order declared.
69
104
  def ordered(steps, label)
70
- numbered = steps.count { |step| step.key?(:order) }
71
- return steps.map { |step| step.except(:order) } if numbered.zero?
105
+ numbered = steps.count { |step| step.key?(:priority) }
106
+ return steps if numbered.zero?
72
107
 
73
108
  if numbered < steps.size
74
- raise ConfigurationError, "#{label}: give every step an order: or none (#{numbered} of #{steps.size} have one)"
109
+ raise ConfigurationError,
110
+ "#{label}: give every step a priority: or none (#{numbered} of #{steps.size} have one)"
75
111
  end
76
112
 
77
- orders = steps.map { |step| step[:order] }
78
- duplicates = orders.tally.select { |_, count| count > 1 }.keys
79
- raise ConfigurationError, "#{label}: duplicate order: #{duplicates.join(', ')}" if duplicates.any?
113
+ steps.each_with_index.sort_by { |step, index| [step[:priority], index] }.map(&:first)
114
+ end
115
+
116
+ # Runs of steps with the same priority. A step without one is a tier of its own.
117
+ def tiers(path)
118
+ path.slice_when { |before, after| before[:priority].nil? || before[:priority] != after[:priority] }
119
+ end
120
+
121
+ def rotates?(tier)
122
+ tier.size > 1 && tier.none? { |step| step[:rotate] == false }
123
+ end
80
124
 
81
- steps.sort_by { |step| step[:order] }.map { |step| step.except(:order) }
125
+ def rotated(path, turn)
126
+ tiers(path).flat_map { |tier| rotates?(tier) ? tier.rotate(turn % tier.size) : tier }
82
127
  end
83
128
 
84
- def normalize_step(step, label)
129
+ # extras: names more keys a step Hash may carry for its caller (a judge's type:, say). They are kept as given,
130
+ # for the caller to check.
131
+ def normalize_step(step, label, extras: [])
85
132
  case step
86
133
  when String, Symbol
87
134
  { model: model_name(step, label), provider: nil, params: nil, attempts: 1, forward_rejected: true }
88
- when Hash then normalize_hash(step, label)
135
+ when Hash then normalize_hash(step, label, extras)
89
136
  else
90
137
  raise ConfigurationError, "#{label}: each step must be a model name or a Hash with model:, got #{step.inspect}"
91
138
  end
92
139
  end
93
140
 
94
- def normalize_hash(step, label)
141
+ def normalize_hash(step, label, extras)
95
142
  step = step.transform_keys(&:to_sym)
96
- unknown = step.keys - STEP_KEYS
143
+ unknown = step.keys - STEP_KEYS - extras
97
144
  raise ConfigurationError, "#{label}: unknown step option(s) #{unknown.join(', ')}" if unknown.any?
98
145
  unless step[:model].is_a?(String) || step[:model].is_a?(Symbol)
99
146
  raise ConfigurationError, "#{label}: a step needs a model: name, got #{step.inspect}"
@@ -104,10 +151,17 @@ module Squishling
104
151
  unless attempts.is_a?(Integer) && attempts.positive?
105
152
  raise ConfigurationError, "#{label} (#{model}): attempts: must be a positive Integer, got #{attempts.inspect}"
106
153
  end
107
- if step.key?(:order) && !step[:order].is_a?(Integer)
108
- raise ConfigurationError, "#{label} (#{model}): order: must be an Integer, got #{step[:order].inspect}"
154
+
155
+ step = priority_alias(step, label, model)
156
+ if step.key?(:priority) && !step[:priority].is_a?(Integer)
157
+ raise ConfigurationError, "#{label} (#{model}): priority: must be an Integer, got #{step[:priority].inspect}"
158
+ end
159
+ if step.key?(:rotate) && ![true, false].include?(step[:rotate])
160
+ raise ConfigurationError, "#{label} (#{model}): rotate: must be true or false, got #{step[:rotate].inspect}"
109
161
  end
110
162
 
163
+ check_include_when!(step, label, model)
164
+ check_retry_temperature!(step, label, model)
111
165
  forward_rejected = step.fetch(:forward_rejected, true)
112
166
  unless [true, false].include?(forward_rejected)
113
167
  raise ConfigurationError,
@@ -116,7 +170,43 @@ module Squishling
116
170
 
117
171
  params = step[:params] && Params.normalize(step[:params], "#{label} (#{model}) params")
118
172
  normalized = { model:, provider: step[:provider], params:, attempts:, forward_rejected: }
119
- step.key?(:order) ? normalized.merge(order: step[:order]) : normalized
173
+ normalized.merge(step.slice(:priority, :rotate, :include_when, :retry_temperature, *extras))
174
+ end
175
+
176
+ def check_include_when!(step, label, model)
177
+ return if !step.key?(:include_when) || step[:include_when].is_a?(Proc)
178
+
179
+ raise ConfigurationError,
180
+ "#{label} (#{model}): include_when: must be a Proc, got #{step[:include_when].inspect}"
181
+ end
182
+
183
+ def valid_temperature?(temperature)
184
+ temperature.is_a?(Numeric) && temperature.real? && temperature.finite? && !temperature.negative?
185
+ end
186
+
187
+ def check_retry_temperature!(step, label, model)
188
+ temperature = step[:retry_temperature]
189
+ return if !step.key?(:retry_temperature) || valid_temperature?(temperature)
190
+
191
+ raise ConfigurationError,
192
+ "#{label} (#{model}): retry_temperature: must be a finite Numeric of 0 or more, got #{temperature.inspect}"
193
+ end
194
+
195
+ # A step used on its own (a judge, a harness's step) can't take part in a tier or be left out of a call, so it
196
+ # takes no priority:, rotate:, or include_when:. who names it for the message.
197
+ def single_step!(step, who)
198
+ return unless step.key?(:priority) || step.key?(:rotate) || step.key?(:include_when)
199
+
200
+ raise ConfigurationError, "#{who} takes no priority:, rotate:, or include_when:"
201
+ end
202
+
203
+ # order: is the deprecated name of priority:.
204
+ def priority_alias(step, label, model)
205
+ return step unless step.key?(:order)
206
+ raise ConfigurationError, "#{label} (#{model}): give priority: or order:, not both" if step.key?(:priority)
207
+
208
+ warn "[Squishling] #{label} (#{model}): order: is deprecated, use priority:"
209
+ step.except(:order).merge(priority: step[:order])
120
210
  end
121
211
 
122
212
  def model_name(model, label)
@@ -0,0 +1,64 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Squishling
4
+ # Classes by name, for the things that can be registered (harnesses, adapters): the built-ins, registered as they
5
+ # load, and the developer's own. Shared across threads, so every read and write is under the lock.
6
+ class NamedRegistry
7
+ NAME = /\A[a-z][a-z0-9_]*\z/
8
+
9
+ # kind names what is registered ("harness"), for messages.
10
+ def initialize(kind)
11
+ @kind = kind
12
+ @lock = Mutex.new
13
+ @built_in = {}
14
+ @custom = {}
15
+ end
16
+
17
+ def register_built_in(name, klass)
18
+ @lock.synchronize { @built_in[name] = klass }
19
+ end
20
+
21
+ # The name as a Symbol. Raises ConfigurationError when it isn't a Symbol or String of lowercase letters, digits
22
+ # and underscores, starting with a letter.
23
+ def checked_name(name)
24
+ unless name.is_a?(Symbol) || name.is_a?(String)
25
+ raise ConfigurationError, "#{@kind} name must be a Symbol or String, got #{name.inspect}"
26
+ end
27
+
28
+ name = name.to_sym
29
+ return name if NAME.match?(name)
30
+
31
+ raise ConfigurationError,
32
+ "#{@kind} name #{name.inspect} must be lowercase letters, digits, and underscores, starting with a letter"
33
+ end
34
+
35
+ # Registers klass under name (already checked), unless the name is taken.
36
+ def add(name, klass)
37
+ @lock.synchronize do
38
+ raise ConfigurationError, "#{@kind} #{name.inspect} is already registered" if taken?(name)
39
+
40
+ @custom[name] = klass
41
+ end
42
+ klass
43
+ end
44
+
45
+ def fetch(name)
46
+ @lock.synchronize { @built_in[name] || @custom[name] }
47
+ end
48
+
49
+ def names
50
+ @lock.synchronize { @built_in.keys + @custom.keys }
51
+ end
52
+
53
+ # Forgets the registered classes; the built-ins stay.
54
+ def reset!
55
+ @lock.synchronize { @custom.clear }
56
+ end
57
+
58
+ private
59
+
60
+ def taken?(name)
61
+ @built_in.key?(name) || @custom.key?(name)
62
+ end
63
+ end
64
+ end
@@ -0,0 +1,109 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Squishling
4
+ # The around-style hook for tools that must see an LLM request while it is made (a tracing span, a Coolhand
5
+ # declared workload), which squawk's after-the-fact report can't do. An observer is any object responding to
6
+ # around(call), which yields once to make the request. See docs/observers.md.
7
+ #
8
+ # Observers get call metadata (Call) and, back from the yield, the request's token Usage: never the model's
9
+ # output or the method's inputs.
10
+ module Observer
11
+ # What an observer is told about one provider request. handed_off is true when squish! sent the call to the
12
+ # LLM (typically from a rescue) and false when squish_when or a missing implementation did. role is nil for an
13
+ # escalation attempt, :sample (with sample "a", "b", ...) or :judge, like squawk's metadata. attempt and
14
+ # attempts count the (expanded) escalation path the request belongs to. request is :chat, or :judgment for a
15
+ # System One judge.
16
+ Call = Data.define(:label, :purpose, :harness, :handed_off, :role, :sample, :attempt, :attempts, :model, :provider,
17
+ :request)
18
+
19
+ # What an observer's block returns: the request's token usage and the model that answered, never the content.
20
+ # A count the provider didn't report is nil.
21
+ Usage = Data.define(:input_tokens, :output_tokens, :model)
22
+
23
+ BUILT_IN = %i[coolhand].freeze
24
+
25
+ module_function
26
+
27
+ # The observer itself, nil (auto-detect), or false (none). A built-in's name (:coolhand) or anything that
28
+ # responds to around is accepted, checked here so a mistake fails at setup, not inside the first request.
29
+ def validate(observer, where)
30
+ return observer if observer.nil? || observer == false
31
+
32
+ if observer.is_a?(Symbol)
33
+ return observer if BUILT_IN.include?(observer)
34
+
35
+ raise ConfigurationError, "#{where} observer must be one of #{BUILT_IN.map(&:inspect).join(', ')}, " \
36
+ "an object that responds to around, or false, got #{observer.inspect}"
37
+ end
38
+ return observer if observer.respond_to?(:around)
39
+
40
+ raise ConfigurationError, "#{where} observer must respond to around(call) { ... } (or be false), " \
41
+ "got #{observer.inspect}"
42
+ end
43
+
44
+ # The observer that wraps a request: the configured one, the Coolhand observer when nothing is configured and
45
+ # Coolhand is loaded and set up, or nil.
46
+ def resolve(configured)
47
+ case configured
48
+ when false then nil
49
+ when nil then Observers::Coolhand.new if Observers::Coolhand.available?
50
+ when Symbol then Observers::Coolhand.new
51
+ else configured
52
+ end
53
+ end
54
+
55
+ # Runs the request inside the resolved observer, if there is one. The block runs on the calling thread, so
56
+ # for samples it is the worker thread that makes the request. The observer gets the request's Usage back from
57
+ # its yield, not the response, so the model's output can't leave the process through it, and what it returns
58
+ # is ignored, and so is any rescue of the request's error: the error is raised again. An observer that never
59
+ # yields would make the request vanish, and one that yields twice would send it twice: both are setup mistakes.
60
+ def around(call)
61
+ observer = resolve(Squishling.config.observer)
62
+ return yield unless observer
63
+
64
+ yielded = false
65
+ response = nil
66
+ failure = nil
67
+ observer.around(call) do
68
+ if yielded
69
+ failure = ConfigurationError.new("#{call.label}: #{observer.class}#around must yield only once")
70
+ raise failure
71
+ end
72
+
73
+ yielded = true
74
+ begin
75
+ response = yield
76
+ rescue StandardError => e
77
+ failure = e
78
+ raise
79
+ end
80
+ usage(response)
81
+ end
82
+ unless yielded
83
+ raise ConfigurationError,
84
+ "#{call.label}: the #{observer.class} observer's around must yield to make the request"
85
+ end
86
+
87
+ raise failure if failure # an observer can't swallow the request's error: Squishling's taxonomy holds
88
+
89
+ response
90
+ end
91
+
92
+ # An Adapter::Response carries its token counts in a Hash; a judgment (see Adapter::Base#judge) may carry them in
93
+ # an object with input and output instead. Shared with Budget::Tally; not part of the public API.
94
+ def usage(response)
95
+ tokens = response.respond_to?(:tokens) ? response.tokens : nil
96
+ model = response.respond_to?(:model) ? response.model : nil
97
+ Usage.new(input_tokens: token_count(tokens, :input_tokens, :input),
98
+ output_tokens: token_count(tokens, :output_tokens, :output),
99
+ model: model.respond_to?(:id) ? model.id : model)
100
+ end
101
+
102
+ def token_count(tokens, key, reader)
103
+ return tokens[key] if tokens.is_a?(Hash)
104
+
105
+ tokens.respond_to?(reader) ? tokens.public_send(reader) : nil
106
+ end
107
+ private_class_method :token_count
108
+ end
109
+ end