squishling 0.4.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +149 -0
  3. data/README.md +26 -10
  4. data/docs/adapters.md +257 -0
  5. data/docs/configuration.md +54 -19
  6. data/docs/failures.md +170 -22
  7. data/docs/harnesses.md +39 -15
  8. data/docs/measuring-tokens.md +12 -6
  9. data/docs/observers.md +3 -3
  10. data/docs/routing.md +14 -3
  11. data/docs/schemas.md +24 -2
  12. data/docs/testing.md +187 -0
  13. data/docs/verifying.md +86 -0
  14. data/lib/squishling/adapter/error_body.rb +54 -0
  15. data/lib/squishling/adapter.rb +328 -0
  16. data/lib/squishling/adapters/openai_compatible.rb +311 -0
  17. data/lib/squishling/adapters/openai_options.rb +114 -0
  18. data/lib/squishling/adapters/openai_transport.rb +123 -0
  19. data/lib/squishling/adapters/ruby_llm.rb +229 -0
  20. data/lib/squishling/budget.rb +71 -0
  21. data/lib/squishling/class_methods.rb +43 -7
  22. data/lib/squishling/class_registry.rb +63 -0
  23. data/lib/squishling/configuration.rb +58 -6
  24. data/lib/squishling/definition.rb +90 -16
  25. data/lib/squishling/errors.rb +94 -7
  26. data/lib/squishling/harness/judged_squishsum.rb +2 -2
  27. data/lib/squishling/harness/option_types.rb +2 -1
  28. data/lib/squishling/harness/registry.rb +8 -36
  29. data/lib/squishling/invoker.rb +44 -14
  30. data/lib/squishling/judge.rb +16 -3
  31. data/lib/squishling/llm_client.rb +66 -132
  32. data/lib/squishling/model_path.rb +60 -13
  33. data/lib/squishling/named_registry.rb +64 -0
  34. data/lib/squishling/observer.rb +13 -4
  35. data/lib/squishling/output_check.rb +13 -9
  36. data/lib/squishling/params.rb +7 -24
  37. data/lib/squishling/railtie.rb +10 -0
  38. data/lib/squishling/result.rb +50 -16
  39. data/lib/squishling/rotation.rb +5 -0
  40. data/lib/squishling/router.rb +3 -2
  41. data/lib/squishling/rspec.rb +18 -0
  42. data/lib/squishling/run/messages.rb +40 -3
  43. data/lib/squishling/run/rejections.rb +43 -0
  44. data/lib/squishling/run/reports.rb +63 -0
  45. data/lib/squishling/run.rb +60 -53
  46. data/lib/squishling/schema.rb +50 -9
  47. data/lib/squishling/secret.rb +45 -0
  48. data/lib/squishling/squawk.rb +37 -4
  49. data/lib/squishling/tasks/squishling.rake +18 -0
  50. data/lib/squishling/testing/adapter.rb +194 -0
  51. data/lib/squishling/testing/checks.rb +39 -0
  52. data/lib/squishling/testing/direct.rb +41 -0
  53. data/lib/squishling/testing/errors.rb +18 -0
  54. data/lib/squishling/testing/fixtures.rb +206 -0
  55. data/lib/squishling/testing/helpers.rb +71 -0
  56. data/lib/squishling/testing/minitest.rb +32 -0
  57. data/lib/squishling/testing/observer.rb +31 -0
  58. data/lib/squishling/testing/reply.rb +60 -0
  59. data/lib/squishling/testing/request.rb +75 -0
  60. data/lib/squishling/testing/rspec.rb +30 -0
  61. data/lib/squishling/testing/script.rb +189 -0
  62. data/lib/squishling/testing/state.rb +74 -0
  63. data/lib/squishling/testing/stubs.rb +112 -0
  64. data/lib/squishling/testing.rb +126 -0
  65. data/lib/squishling/verification.rb +39 -0
  66. data/lib/squishling/verifier.rb +179 -0
  67. data/lib/squishling/version.rb +1 -1
  68. data/lib/squishling/wrapper.rb +19 -1
  69. data/lib/squishling.rb +60 -4
  70. metadata +43 -21
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Squishling
4
+ # Every class that includes Squishling (and each subclass of one), so they can be verified ahead of runtime.
5
+ # Held weakly, so classes replaced by code reloading (e.g. Rails in development) can be collected; a class that is
6
+ # replaced but not yet collected is left out of the list. Shared across threads, so every access is under the lock.
7
+ module ClassRegistry
8
+ LOCK = Mutex.new
9
+ GONE = Object.new.freeze
10
+ UNKNOWN = Object.new.freeze
11
+ private_constant :LOCK, :GONE, :UNKNOWN
12
+
13
+ @classes = ObjectSpace::WeakMap.new
14
+
15
+ class << self
16
+ def register(klass)
17
+ LOCK.synchronize { @classes[klass] = true }
18
+ klass
19
+ end
20
+
21
+ # The live classes, in no particular order, as a snapshot.
22
+ def classes
23
+ LOCK.synchronize { @classes.keys }.reject { |klass| replaced?(klass) }
24
+ end
25
+
26
+ # Forgets every class, e.g. between specs.
27
+ def reset!
28
+ LOCK.synchronize { @classes = ObjectSpace::WeakMap.new }
29
+ end
30
+
31
+ private
32
+
33
+ # A named class whose constant is gone or now holds another object (a reload has replaced it). Anonymous
34
+ # classes have no constant to go stale, and a name that can't be walked (under an anonymous module, a temporary
35
+ # name) is taken as live.
36
+ def replaced?(klass)
37
+ name = Module.instance_method(:name).bind_call(klass) # the class may define its own .name
38
+ return false unless name
39
+
40
+ found = lookup(name)
41
+ !found.equal?(UNKNOWN) && !found.equal?(klass)
42
+ end
43
+
44
+ def lookup(name)
45
+ name.split("::").reduce(Object) do |scope, constant|
46
+ found = child(scope, constant)
47
+ return found if found.equal?(GONE) || found.equal?(UNKNOWN)
48
+
49
+ found
50
+ end
51
+ end
52
+
53
+ # The constant in scope, without autoloading it: GONE when it is missing or a pending autoload (Zeitwerk resets
54
+ # a reloaded constant to one, and loading it would run app code), UNKNOWN when the name can't be walked.
55
+ def child(scope, constant)
56
+ return UNKNOWN unless scope.is_a?(Module) && constant.match?(/\A[A-Z]\w*\z/)
57
+ return GONE if scope.autoload?(constant, false) || !scope.const_defined?(constant, false)
58
+
59
+ scope.const_get(constant, false)
60
+ end
61
+ end
62
+ end
63
+ end
@@ -3,7 +3,7 @@
3
3
  module Squishling
4
4
  class Configuration
5
5
  # Model used by every squishling class that doesn't declare its own (a single attempt).
6
- # When neither this nor default_escalation is set, RubyLLM's own default model is used.
6
+ # When neither this nor default_escalation is set, the adapter's own default model is used.
7
7
  attr_reader :default_model
8
8
 
9
9
  # Models tried in order by every squishling class that doesn't declare its own, e.g.
@@ -14,13 +14,14 @@ module Squishling
14
14
  # The validated default_model or default_escalation steps (nil when neither is set).
15
15
  attr_reader :default_model_path
16
16
 
17
- # Provider for default_model/default_escalation steps that don't name their own (e.g. :openai). Only
18
- # needed for models missing from RubyLLM's registry.
17
+ # Provider for default_model/default_escalation steps that don't name their own (e.g. :openai). What it means is
18
+ # up to the adapter: the RubyLLM adapter needs it only for models missing from RubyLLM's registry, and the
19
+ # OpenAI-compatible adapter ignores it.
19
20
  attr_accessor :default_provider
20
21
 
21
22
  # Generation params applied to every call, overridable per class and per method.
22
- # :temperature and :thinking ({ effort:, budget: }) map to RubyLLM's with_temperature and
23
- # with_thinking; any other key (top_p, max_tokens, seed, ...) is passed to the provider as-is.
23
+ # :temperature, :max_output_tokens and :thinking ({ effort:, budget: }) are portable and translated by the
24
+ # adapter for its provider; any other key (top_p, seed, ...) is passed to the provider as-is.
24
25
  attr_reader :default_params
25
26
 
26
27
  # How every squishling class that doesn't declare its own harness uses its escalation (see Harness):
@@ -28,19 +29,38 @@ module Squishling
28
29
  # its options.
29
30
  attr_reader :default_harness
30
31
 
32
+ # Most output tokens, and most input tokens, one squished call may use across all its requests (nil, the default,
33
+ # sets no limit). A call that has used this many stops with a BudgetExhaustedError instead of making another
34
+ # attempt; overridable per class, method and squish! call (see Budget).
35
+ attr_reader :default_max_total_output_tokens, :default_max_total_input_tokens
36
+
31
37
  # Optional Logger for routing and escalation decisions.
32
38
  attr_accessor :logger
33
39
 
34
- # Optional callable run after every LLM attempt with the raw output (see Squawk), e.g.
40
+ # Optional callable run after each LLM attempt (narrowed by squawk_on), and once more when a token budget or a
41
+ # disagreement stops the call, with the raw output (see Squawk), e.g.
35
42
  # ->(output:, metadata:, error:) { Tracer.record(output, metadata, error) }
36
43
  attr_reader :squawk
37
44
 
45
+ # Which attempts reach the squawk hook: :all (the default), :rejections (attempts that errored), :retries (those
46
+ # plus any accepted attempt after the first), :final (the last attempt the escalation allows), or a list of
47
+ # them. Also settable per class and per method.
48
+ attr_reader :squawk_on
49
+
38
50
  # Optional around-style hook wrapped around every provider request (see Observer): an object responding to
39
51
  # around(call) { ... }, or :coolhand. nil (the default) uses the Coolhand observer when the coolhand gem is
40
52
  # loaded and configured, and nothing otherwise; false never wraps.
41
53
  attr_reader :observer
42
54
 
55
+ # How Squishling reaches a model (see Adapter): :ruby_llm, :openai_compatible, an adapter you registered, a Hash
56
+ # with type: and that adapter's options, or an adapter instance. nil (the default) uses RubyLLM when the ruby_llm
57
+ # gem is available. Built when assigned, so a mistake fails at setup rather than on the first request.
58
+ attr_reader :adapter
59
+
43
60
  def initialize
61
+ @adapter = nil
62
+ @detected_adapter = nil
63
+ @adapter_lock = Mutex.new
44
64
  @observer = nil
45
65
  @default_model = nil
46
66
  @default_escalation = nil
@@ -50,6 +70,7 @@ module Squishling
50
70
  @default_harness = nil
51
71
  @logger = nil
52
72
  @squawk = nil
73
+ @squawk_on = :all
53
74
  end
54
75
 
55
76
  def default_model=(model)
@@ -64,10 +85,41 @@ module Squishling
64
85
  @default_escalation = escalation
65
86
  end
66
87
 
88
+ def adapter=(adapter)
89
+ built = adapter.nil? ? nil : Adapter.build(adapter, "Squishling.configure")
90
+ @adapter_lock.synchronize do
91
+ @adapter = built
92
+ @detected_adapter = nil
93
+ end
94
+ end
95
+
96
+ # The adapter requests go through: the configured one, else RubyLLM's (when its gem is available) built on first
97
+ # use. Raises ConfigurationError when there is neither.
98
+ def resolved_adapter
99
+ @adapter_lock.synchronize { @adapter || (@detected_adapter ||= Adapter.detect) }
100
+ end
101
+
102
+ def default_max_total_output_tokens=(tokens)
103
+ @default_max_total_output_tokens = Budget.build("default_max_total_output_tokens", output: tokens)[:output]
104
+ end
105
+
106
+ def default_max_total_input_tokens=(tokens)
107
+ @default_max_total_input_tokens = Budget.build("default_max_total_input_tokens", input: tokens)[:input]
108
+ end
109
+
110
+ # The configured limits as a Budget level.
111
+ def default_budget
112
+ { output: default_max_total_output_tokens, input: default_max_total_input_tokens }.compact
113
+ end
114
+
67
115
  def squawk=(hook)
68
116
  @squawk = Squawk.validate(hook, "Squishling.configure")
69
117
  end
70
118
 
119
+ def squawk_on=(on)
120
+ @squawk_on = Squawk.validate_on(on, "Squishling.configure") || :all
121
+ end
122
+
71
123
  def observer=(observer)
72
124
  Observer.validate(observer, "Squishling.configure")
73
125
  # A named observer is built now, so a missing gem fails at setup rather than on the first request.
@@ -7,11 +7,14 @@ module Squishling
7
7
  # Matches the keys a squish! context: Hash may use.
8
8
  NAME_KEY = ->(key) { key.is_a?(String) || key.is_a?(Symbol) }
9
9
 
10
+ # One attempt on RubyLLM's default model, when no level declares a model.
11
+ DEFAULT_STEP = { model: nil, attempts: 1, forward_rejected: true }.freeze
12
+
10
13
  attr_reader :klass, :name, :call_context, :handed_off
11
14
 
12
15
  def initialize(klass:, name:, purpose: nil, append_to_purpose: nil, output_schema: nil, model: nil,
13
16
  provider: nil, params: nil, harness: nil, predicate: nil, fallback: nil, validator: nil, squawk: nil,
14
- call_context: {}, handed_off: false)
17
+ squawk_on: nil, budget: nil, call_context: {}, handed_off: false)
15
18
  @klass = klass
16
19
  @name = name
17
20
  @purpose = purpose
@@ -25,6 +28,8 @@ module Squishling
25
28
  @fallback = fallback
26
29
  @validator = validator
27
30
  @squawk = squawk
31
+ @squawk_on = squawk_on
32
+ @budget = budget || {}
28
33
  @call_context = call_context
29
34
  @handed_off = handed_off
30
35
  end
@@ -32,7 +37,7 @@ module Squishling
32
37
  # This definition with one call's overrides on top. The output schema, predicate, fallback, and validator
33
38
  # can't be overridden: the call must still return the method's result type.
34
39
  def for_call(purpose: nil, append_to_purpose: nil, context: nil, model: nil, escalation: nil,
35
- provider: nil, params: nil, harness: nil)
40
+ provider: nil, params: nil, harness: nil, max_total_output_tokens: nil, max_total_input_tokens: nil)
36
41
  call_path = ModelPath.declare(model, escalation, "#{label} squish!")
37
42
  raise ConfigurationError, "#{label}: squish! provider: needs a model: or escalation:" if provider && !call_path
38
43
  unless context.nil? || (context.is_a?(Hash) && context.each_key.all?(NAME_KEY))
@@ -42,15 +47,16 @@ module Squishling
42
47
  appended = Appendices.normalize(append_to_purpose, "#{label} squish!") unless append_to_purpose.nil?
43
48
  call_params = params && Params.normalize(params, "#{label} squish! params")
44
49
  call_harness = Harness.normalize(harness, "#{label} squish!") unless harness.nil?
50
+ call_budget = Budget.build("#{label} squish!", output: max_total_output_tokens, input: max_total_input_tokens)
45
51
  self.class.new(
46
52
  klass:, name:, output_schema: @output_schema, predicate: @predicate, fallback: @fallback,
47
- validator: @validator, squawk: @squawk,
53
+ validator: @validator, squawk: @squawk, squawk_on: @squawk_on,
48
54
  purpose: purpose || @purpose,
49
55
  append_to_purpose: [*@append_to_purpose, *appended],
50
56
  model: call_path || @model, provider: call_path ? provider : @provider,
51
57
  # merge, not Params.resolve: a nil at the method level must still unset the class's key.
52
58
  params: call_params ? (@params || {}).merge(call_params) : @params,
53
- harness: call_harness || @harness,
59
+ harness: call_harness || @harness, budget: @budget.merge(call_budget),
54
60
  call_context: @call_context.merge((context || {}).transform_keys(&:to_sym)),
55
61
  handed_off: true # only squish! builds a definition this way; it is not part of a declared one
56
62
  )
@@ -62,7 +68,7 @@ module Squishling
62
68
 
63
69
  # The system prompt: the purpose, then each append_to_purpose section.
64
70
  def purpose(receiver)
65
- value = @purpose || klass.purpose
71
+ value = declared_purpose
66
72
  value = receiver.instance_exec(&value) if value.is_a?(Proc)
67
73
  return value if value.nil? || value.empty?
68
74
 
@@ -79,16 +85,48 @@ module Squishling
79
85
  end
80
86
 
81
87
  # One ModelPath::Step per attempt, from the first level (method, class, or config) that declares a model
82
- # or escalation, or a single attempt on RubyLLM's default model. A provider travels with the model declared
88
+ # or escalation, or a single attempt on the adapter's default model. A provider travels with the model declared
83
89
  # at the same level, so a per-method Anthropic model never inherits a class-level OpenAI provider. Models
84
- # tied on priority are rotated by this method's call count (see Rotation).
85
- def escalation_path
90
+ # tied on priority are rotated by this method's call count (see Rotation). Steps whose include_when: is falsy
91
+ # for this call's inputs are left out, so a tier rotates only between its eligible models. advance: false looks
92
+ # at the turn the next call will get without counting this one (a prompt preview).
93
+ def escalation_path(receiver, inputs, advance: true)
94
+ declared, provider = declared_path
95
+ eligible = ModelPath.eligible(declared, receiver, inputs, label)
96
+ # Counted when the declared path rotates (so a call that excludes some models doesn't skew the others'
97
+ # rotation) or when this call's eligible steps do (an excluded rotate: false step no longer pins its tier).
98
+ rotates = ModelPath.rotating?(declared) || ModelPath.rotating?(eligible)
99
+ turn = if rotates
100
+ advance ? Rotation.advance(klass, name) : Rotation.peek(klass, name)
101
+ else
102
+ 0
103
+ end
104
+ ModelPath.steps(eligible, provider:, params:, turn:)
105
+ end
106
+
107
+ # The steps and provider of the first level (method, class, or config) that declares a model or escalation, or
108
+ # the default step, before any call's include_when: filtering.
109
+ def declared_path
86
110
  config = Squishling.config
87
111
  entries, provider = [[@model, @provider], [klass.squishling_model_path, klass.squishling_provider],
88
112
  [config.default_model_path, config.default_provider]].find(&:first)
89
- entries ||= [{ model: nil, attempts: 1, forward_rejected: true }]
90
- turn = ModelPath.rotating?(entries) ? Rotation.advance(klass, name) : 0
91
- ModelPath.steps(entries, provider:, params:, turn:)
113
+ [entries || [DEFAULT_STEP], provider]
114
+ end
115
+
116
+ # Whether a purpose is declared for this method or its class (a Proc one can't be evaluated without a receiver).
117
+ def purpose_declared?
118
+ value = declared_purpose
119
+ value.is_a?(Proc) || !(value.nil? || value.empty?)
120
+ end
121
+
122
+ # The names the method's inputs are sent under (see #bind_arguments), and whether any keyword may be an input:
123
+ # there is no implementation to name them, it takes **, or has a parameter with no name to send it under.
124
+ def input_names
125
+ names = parameters.filter_map { |type, param| input_name(type, param) }
126
+ any_keyword = !implemented? || parameters.any? do |type, param|
127
+ type == :keyrest || (Squawk::POSITIONAL_TYPES.include?(type) && param.nil?)
128
+ end
129
+ [names, any_keyword]
92
130
  end
93
131
 
94
132
  # How the escalation is used (see Harness): the method's, the class's, the configured default, or :escalation.
@@ -101,23 +139,48 @@ module Squishling
101
139
  Params.resolve(Squishling.config.default_params, klass.squishling_params, @params)
102
140
  end
103
141
 
142
+ # The token limits of one call (see Budget): config defaults, overridden limit by limit by the class, then by the
143
+ # method (and a squish! call).
144
+ def budget
145
+ Budget.resolve(Squishling.config.default_budget, klass.squishling_budget, @budget)
146
+ end
147
+
104
148
  # Extra output checks run on schema-valid LLM results (see ClassMethods#squish_validate).
105
149
  def validator
106
150
  @validator || klass.squishling_validator
107
151
  end
108
152
 
153
+ # The squish_fallback handler: the method's, else the class's (nil when there is none).
154
+ def fallback
155
+ @fallback || klass.squishling_fallback
156
+ end
157
+
109
158
  # The observability hook: the method's, else the class's, else the configured one. `false` at a level
110
159
  # silences the levels above it.
111
160
  def squawk
112
161
  [@squawk, klass.squishling_squawk, Squishling.config.squawk].find { |hook| !hook.nil? } || nil
113
162
  end
114
163
 
164
+ # Which attempts reach the hook (see Squawk): the method's, else the class's, else the configured one.
165
+ def squawk_on
166
+ [@squawk_on, klass.squishling_squawk_on, Squishling.config.squawk_on].find { |on| !on.nil? }
167
+ end
168
+
115
169
  def context_names
116
170
  klass.squishling_context_names
117
171
  end
118
172
 
173
+ # The squish_when predicate: the method's, else the class's.
174
+ def predicate
175
+ @predicate || klass.squishling_predicate
176
+ end
177
+
178
+ # Whether the method has a Ruby implementation (otherwise every call goes to the LLM).
179
+ def implemented?
180
+ !Wrapper.implementation(klass.instance_method(name)).nil?
181
+ end
182
+
119
183
  def squish?(receiver, inputs)
120
- predicate = @predicate || klass.squishling_predicate
121
184
  return false unless predicate
122
185
 
123
186
  receiver.instance_exec(**inputs, &predicate) ? true : false
@@ -128,7 +191,7 @@ module Squishling
128
191
  def invoke_llm(receiver, inputs)
129
192
  Invoker.new(self, receiver, inputs).call
130
193
  rescue InvalidOutputError, LLMError => e
131
- handler = @fallback || klass.squishling_fallback
194
+ handler = fallback
132
195
  raise unless handler
133
196
 
134
197
  Squishling.config.logger&.warn("[Squishling] #{label} LLM failed, using fallback: #{e.message}")
@@ -148,7 +211,7 @@ module Squishling
148
211
 
149
212
  attrs = attrs.to_h if attrs.is_a?(Result::Instance)
150
213
  data, errors = jsonify_return(attrs)
151
- errors = schema.validate(data) if errors.empty?
214
+ errors = InvalidOutputError::Detail.from(schema.validate(data), :schema) if errors.empty?
152
215
  raise InvalidOutputError.new(errors:, raw: attrs, source: "Deterministic") if errors.any?
153
216
 
154
217
  schema.build(data, squished:)
@@ -174,7 +237,7 @@ module Squishling
174
237
  unnamed += 1
175
238
  end
176
239
  when :rest
177
- bound[param.nil? || param == :* ? :args : param] = positional.shift(positional.size)
240
+ bound[input_name(type, param)] = positional.shift(positional.size)
178
241
  end
179
242
  end
180
243
  positional.each_with_index { |value, index| bound[:"arg#{unnamed + index}"] = value }
@@ -188,7 +251,18 @@ module Squishling
188
251
  def jsonify_return(attrs)
189
252
  [Schema.jsonify(attrs), []]
190
253
  rescue JSON::GeneratorError, JSON::NestingError => e
191
- [nil, [e.message]]
254
+ [nil, [InvalidOutputError::Detail.new(e.message, :parse)]]
255
+ end
256
+
257
+ def declared_purpose
258
+ @purpose || klass.purpose
259
+ end
260
+
261
+ def input_name(type, param)
262
+ case type
263
+ when :req, :opt, :key, :keyreq then param
264
+ when :rest then param.nil? || param == :* ? :args : param
265
+ end
192
266
  end
193
267
 
194
268
  # The implementation's parameters, beneath the prepended wrappers.
@@ -7,19 +7,88 @@ module Squishling
7
7
  # missing API key). Never retried and never passed to squish_fallback.
8
8
  class ConfigurationError < Error; end
9
9
 
10
+ # The provider answered a request with HTTP 400 (or another 4xx that isn't a credentials, billing, model, redirect,
11
+ # or too-long-input problem; the adapter decides which, see Adapter::Base#http_failure). Still a ConfigurationError:
12
+ # never retried, escalated, or passed to a fallback. The provider's message is never repeated; the machine-readable
13
+ # status and code its error body carries are, when they are plain tokens (e.g. "INVALID_ARGUMENT",
14
+ # "invalid_request_error"). Both are nil when the body has none.
15
+ class RequestRejectedError < ConfigurationError
16
+ attr_reader :provider_status, :provider_code
17
+
18
+ def initialize(message, provider_status: nil, provider_code: nil)
19
+ @provider_status = provider_status
20
+ @provider_code = provider_code
21
+ super(message)
22
+ end
23
+ end
24
+
25
+ # Squishling.verify! found problems in the squished classes' setup. report holds them all (see Verification::Report).
26
+ class VerificationError < ConfigurationError
27
+ attr_reader :report
28
+
29
+ def initialize(report)
30
+ @report = report
31
+ super(report.to_s)
32
+ end
33
+
34
+ def problems
35
+ report.problems
36
+ end
37
+ end
38
+
10
39
  # Output that didn't match the schema (or a squish_validate check): from the LLM after every attempt in the
11
40
  # escalation, or from the deterministic path.
12
41
  class InvalidOutputError < Error
13
- # models: the model ids tried, one per attempt (nil entries mean RubyLLM's default model).
14
- attr_reader :errors, :raw, :attempts, :models
42
+ # Added to the message (and the log line) when the last reply was cut off at the output token cap.
43
+ CUTOFF_NOTE = "cut off at the output token limit"
44
+
45
+ # One entry of #error_details: the message and where it came from.
46
+ # :schema a JSON Schema error; it can name a key the model chose (a disallowed additional property)
47
+ # :validator your own squish_validate message, verbatim (so it holds model values only if you interpolate them)
48
+ # :parse a message about a response or return value that didn't parse (not valid JSON, empty,
49
+ # a value JSON can't represent)
50
+ # :squishling any other text Squishling or a custom harness wrote
51
+ Detail = Data.define(:message, :source) do
52
+ # One detail per message, all from the same source.
53
+ def self.from(messages, source) = messages.map { |message| new(message, source) }
15
54
 
16
- def initialize(errors:, raw: nil, source: "LLM", attempts: nil, models: nil)
17
- @errors = errors
55
+ def initialize(message:, source:)
56
+ unless Detail::SOURCES.include?(source)
57
+ raise ArgumentError, "unknown error source #{source.inspect} (expected #{Detail::SOURCES.join(', ')})"
58
+ end
59
+
60
+ super
61
+ end
62
+
63
+ # Whether the message can contain text the model chose, so a sink with its own control syntax should escape
64
+ # it. Your own squish_validate messages are never flagged, even if you interpolate result values into them.
65
+ def model_derived? = source == :schema
66
+ end
67
+ Detail::SOURCES = %i[schema validator parse squishling].freeze
68
+
69
+ # models: the model ids tried, one per attempt (nil entries mean the adapter's default model). finish_reason: why
70
+ # the provider stopped on the last attempt (a Symbol such as :stop or :max_tokens; nil when unknown or not from
71
+ # an LLM).
72
+ # errors: Strings, or Details; a String counts as :squishling text.
73
+ attr_reader :errors, :error_details, :raw, :attempts, :models, :finish_reason
74
+
75
+ def initialize(errors:, raw: nil, source: "LLM", attempts: nil, models: nil, finish_reason: nil)
76
+ @error_details = errors.map { |error| error.is_a?(Detail) ? error : Detail.new(error.to_s, :squishling) }
77
+ @error_details.freeze
78
+ @errors = @error_details.map(&:message).freeze
18
79
  @raw = raw
19
80
  @attempts = attempts
20
81
  @models = models
82
+ @finish_reason = finish_reason
21
83
  tries = attempts ? " after #{attempts} attempt#{'s' unless attempts == 1}" : ""
22
- super("#{source} output was invalid#{tries}: #{errors.join('; ')}")
84
+ cutoff = " (#{CUTOFF_NOTE})" if truncated?
85
+ super("#{source} output was invalid#{tries}: #{@errors.join('; ')}#{cutoff}")
86
+ end
87
+
88
+ # Whether the last reply was cut off at the output token cap, so raising max_output_tokens (leaving room for any
89
+ # thinking budget) is the fix rather than changing the prompt or validator.
90
+ def truncated?
91
+ finish_reason == :max_tokens
23
92
  end
24
93
  end
25
94
 
@@ -44,7 +113,25 @@ module Squishling
44
113
  end
45
114
  end
46
115
 
47
- # The LLM call itself failed (rate limit, server error, timeout, connection) after RubyLLM's own
48
- # HTTP retries. The original exception is available as #cause.
116
+ # The call spent its token budget (max_total_output_tokens or max_total_input_tokens) before any attempt was
117
+ # accepted, so no further request was made. limit names the budget that was reached, max its value, and used the
118
+ # tokens spent so far ({ input_tokens:, output_tokens: }). errors starts with the budget line, then the validation
119
+ # errors of the last rejected output, which is in raw (nil when no output had been rejected, e.g. the samples were
120
+ # valid but the judge could not run). Like any InvalidOutputError it goes to squish_fallback.
121
+ class BudgetExhaustedError < InvalidOutputError
122
+ attr_reader :limit, :max, :used
123
+
124
+ def initialize(limit:, max:, used:, errors: [], raw: nil, attempts: nil, models: nil)
125
+ @limit = limit
126
+ @max = max
127
+ @used = used
128
+ spent = used.fetch(:"#{limit.to_s.delete_prefix('max_total_')}")
129
+ super(errors: ["token budget exhausted: #{spent} of #{max} #{limit} used", *errors], raw:, attempts:, models:)
130
+ end
131
+ end
132
+
133
+ # The LLM call itself failed (rate limit, server error, timeout, connection) after the
134
+ # adapter's own HTTP retries. #cause is the adapter's: a sanitized stand-in (Adapter::ProviderFailure) for a provider
135
+ # exception that carries a response body, or the transport error itself when it carries none.
49
136
  class LLMError < Error; end
50
137
  end
@@ -4,10 +4,10 @@ module Squishling
4
4
  class Harness
5
5
  # Like Squishling::Harness::Squishsum, but samples that disagree go to a judge, which picks one or rejects both.
6
6
  # A judge is one step: a model name or a Hash with the escalation step keys (except priority:), plus
7
- # type: (:chat, the default, or :judgment for a System One decision model through RubyLLM.judge) and,
7
+ # type: (:chat, the default, or :judgment for a System One decision model through the adapter's judgments) and,
8
8
  # for :judgment, min_confidence: (the probability the chosen candidate needs).
9
9
  class JudgedSquishsum < Base
10
- # Request keys RubyLLM's judgment protocols own (model and input are already reserved params).
10
+ # Request keys the RubyLLM adapter's judgment protocols own (model and input are already reserved params).
11
11
  JUDGMENT_RESERVED_PARAMS = %i[state questions].freeze
12
12
  private_constant :JUDGMENT_RESERVED_PARAMS
13
13
 
@@ -4,7 +4,8 @@ module Squishling
4
4
  class Harness
5
5
  # The types a harness option can declare (see Base.option). Each checks a given value and returns it
6
6
  # normalized, or raises ConfigurationError, so a bad option fails before any request is made.
7
- # :step a model name or a step Hash (model:, provider:, params:, attempts:, forward_rejected:)
7
+ # :step a model name or a step Hash (model:, provider:, params:, attempts:, forward_rejected:,
8
+ # retry_temperature:)
8
9
  # :proc, :string, :symbol, :integer, :boolean
9
10
  # A harness with a shape of its own declares the type as a callable instead, which gets the value, the
10
11
  # declaration site, and the option's name, and returns the normalized value (or raises ConfigurationError).
@@ -3,18 +3,13 @@
3
3
  module Squishling
4
4
  class Harness
5
5
  # The harness types by name: the built-ins, registered as they load, and your own (Squishling.register_harness).
6
- # Shared across threads, so every read and write is under the lock.
7
6
  module Registry
8
- NAME = /\A[a-z][a-z0-9_]*\z/
9
-
10
- LOCK = Mutex.new
11
- private_constant :LOCK
12
- @built_in = {}
13
- @custom = {}
7
+ STORE = NamedRegistry.new("harness")
8
+ private_constant :STORE
14
9
 
15
10
  class << self
16
11
  def register_built_in(name, klass)
17
- LOCK.synchronize { @built_in[name] = klass }
12
+ STORE.register_built_in(name, klass)
18
13
  end
19
14
 
20
15
  # Registers a harness class (or a block, which becomes the body of a Base subclass) under a name.
@@ -22,46 +17,23 @@ module Squishling
22
17
  raise ConfigurationError, "register_harness takes a class or a block, not both" if klass && block
23
18
  raise ConfigurationError, "register_harness needs a Harness::Base subclass or a block" unless klass || block
24
19
 
25
- name = checked_name(name)
20
+ name = STORE.checked_name(name)
26
21
  klass = Class.new(Base, &block) if block
27
22
  Base.check_registrable!(klass, name)
28
- LOCK.synchronize do
29
- raise ConfigurationError, "harness #{name.inspect} is already registered" if taken?(name)
30
-
31
- @custom[name] = klass
32
- end
33
- klass
23
+ STORE.add(name, klass)
34
24
  end
35
25
 
36
26
  def fetch(name)
37
- LOCK.synchronize { @built_in[name] || @custom[name] }
27
+ STORE.fetch(name)
38
28
  end
39
29
 
40
30
  def names
41
- LOCK.synchronize { @built_in.keys + @custom.keys }
31
+ STORE.names
42
32
  end
43
33
 
44
34
  # Forgets the registered harnesses; the built-ins stay.
45
35
  def reset!
46
- LOCK.synchronize { @custom.clear }
47
- end
48
-
49
- private
50
-
51
- def checked_name(name)
52
- unless name.is_a?(Symbol) || name.is_a?(String)
53
- raise ConfigurationError, "a harness name must be a Symbol or String, got #{name.inspect}"
54
- end
55
-
56
- name = name.to_sym
57
- return name if NAME.match?(name)
58
-
59
- raise ConfigurationError,
60
- "harness name #{name.inspect} must be lowercase letters, digits, and underscores, starting with a letter"
61
- end
62
-
63
- def taken?(name)
64
- @built_in.key?(name) || @custom.key?(name)
36
+ STORE.reset!
65
37
  end
66
38
  end
67
39
  end