squishling 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +133 -1
- data/README.md +63 -15
- data/docs/adapters.md +189 -0
- data/docs/configuration.md +89 -24
- data/docs/coolhand.md +66 -0
- data/docs/failures.md +129 -20
- data/docs/harnesses.md +186 -48
- data/docs/measuring-tokens.md +36 -42
- data/docs/observers.md +80 -0
- data/docs/routing.md +7 -2
- data/docs/schemas.md +33 -6
- data/docs/testing.md +183 -0
- data/docs/verifying.md +84 -0
- data/lib/squishling/adapter/error_body.rb +47 -0
- data/lib/squishling/adapter.rb +275 -0
- data/lib/squishling/adapters/openai_compatible.rb +304 -0
- data/lib/squishling/adapters/openai_options.rb +72 -0
- data/lib/squishling/adapters/ruby_llm.rb +193 -0
- data/lib/squishling/budget.rb +71 -0
- data/lib/squishling/class_methods.rb +43 -7
- data/lib/squishling/class_registry.rb +63 -0
- data/lib/squishling/configuration.rb +72 -8
- data/lib/squishling/definition.rb +95 -17
- data/lib/squishling/errors.rb +97 -12
- data/lib/squishling/harness/base.rb +136 -0
- data/lib/squishling/harness/escalation.rb +22 -0
- data/lib/squishling/harness/judged_squishsum.rb +83 -0
- data/lib/squishling/harness/option_types.rb +94 -0
- data/lib/squishling/harness/registry.rb +41 -0
- data/lib/squishling/harness/squishsum.rb +22 -0
- data/lib/squishling/harness.rb +25 -99
- data/lib/squishling/invoker.rb +28 -233
- data/lib/squishling/judge.rb +78 -21
- data/lib/squishling/llm_client.rb +73 -109
- data/lib/squishling/model_path.rb +117 -27
- data/lib/squishling/named_registry.rb +64 -0
- data/lib/squishling/observer.rb +109 -0
- data/lib/squishling/observers/coolhand.rb +86 -0
- data/lib/squishling/output_check.rb +13 -9
- data/lib/squishling/params.rb +15 -27
- data/lib/squishling/railtie.rb +10 -0
- data/lib/squishling/result.rb +37 -16
- data/lib/squishling/rotation.rb +31 -0
- data/lib/squishling/rspec.rb +18 -0
- data/lib/squishling/run/messages.rb +79 -0
- data/lib/squishling/run/rejections.rb +43 -0
- data/lib/squishling/run.rb +320 -0
- data/lib/squishling/schema.rb +10 -9
- data/lib/squishling/squawk.rb +33 -3
- data/lib/squishling/tasks/squishling.rake +18 -0
- data/lib/squishling/testing/adapter.rb +194 -0
- data/lib/squishling/testing/checks.rb +39 -0
- data/lib/squishling/testing/direct.rb +41 -0
- data/lib/squishling/testing/errors.rb +18 -0
- data/lib/squishling/testing/fixtures.rb +206 -0
- data/lib/squishling/testing/helpers.rb +71 -0
- data/lib/squishling/testing/minitest.rb +32 -0
- data/lib/squishling/testing/observer.rb +31 -0
- data/lib/squishling/testing/reply.rb +60 -0
- data/lib/squishling/testing/request.rb +75 -0
- data/lib/squishling/testing/rspec.rb +30 -0
- data/lib/squishling/testing/script.rb +189 -0
- data/lib/squishling/testing/state.rb +73 -0
- data/lib/squishling/testing/stubs.rb +112 -0
- data/lib/squishling/testing.rb +126 -0
- data/lib/squishling/verification.rb +39 -0
- data/lib/squishling/verifier.rb +179 -0
- data/lib/squishling/version.rb +1 -1
- data/lib/squishling.rb +89 -4
- metadata +53 -21
|
@@ -1,25 +1,41 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Squishling
|
|
4
|
-
#
|
|
4
|
+
# The requests for one squished method, made through the configured adapter (see Adapter). Squishling owns the
|
|
5
|
+
# conversation: each Conversation holds the system prompt, the output schema, and the messages so far, and every
|
|
6
|
+
# request sends all of it to the adapter.
|
|
5
7
|
class LLMClient
|
|
6
|
-
#
|
|
7
|
-
|
|
8
|
-
|
|
8
|
+
# One chat on one escalation step. messages are { role:, content: } Hashes, grown by #ask.
|
|
9
|
+
Conversation = Struct.new(:step, :instructions, :schema, :messages)
|
|
10
|
+
|
|
11
|
+
# purpose, harness (the harness type), and handed_off (squish! sent the call here) are only told to the observer.
|
|
12
|
+
# budget (see Budget) caps the tokens the requests of this client may use between them.
|
|
13
|
+
def initialize(label, purpose:, harness:, handed_off:, budget: {})
|
|
14
|
+
@label = label
|
|
15
|
+
@purpose = purpose
|
|
16
|
+
@harness = harness
|
|
17
|
+
@handed_off = handed_off
|
|
18
|
+
@adapter = Squishling.config.resolved_adapter
|
|
19
|
+
@tally = Budget::Tally.new(budget)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# The budget limit this client's requests have reached ({ limit:, max:, used: }), or nil. Checked before each
|
|
23
|
+
# request, so one already in flight is never cut short.
|
|
24
|
+
def exhausted_budget
|
|
25
|
+
@tally.exhausted
|
|
26
|
+
end
|
|
9
27
|
|
|
10
28
|
# Runs each job on its own thread and returns, in order, each job's [value, exception]. Only provider
|
|
11
29
|
# requests run here: parsing, validation, and the developer's callbacks stay on the caller's thread. Fiber
|
|
12
|
-
# storage
|
|
13
|
-
#
|
|
14
|
-
#
|
|
15
|
-
|
|
16
|
-
# stopped rather than left running unobserved.
|
|
17
|
-
def self.concurrently(jobs)
|
|
30
|
+
# storage is inherited by the threads, and the adapter carries over whatever request context it keeps (see
|
|
31
|
+
# Adapter::Base#capture_context). If the caller is interrupted while waiting (a timeout, Thread#raise), the
|
|
32
|
+
# requests still running are stopped rather than left running unobserved.
|
|
33
|
+
def concurrently(jobs)
|
|
18
34
|
return jobs.map { |job| capture(job) } if jobs.size < 2
|
|
19
35
|
|
|
20
|
-
context =
|
|
36
|
+
context = @adapter.capture_context if @adapter.respond_to?(:capture_context)
|
|
21
37
|
threads = jobs.map do |job|
|
|
22
|
-
thread = Thread.new {
|
|
38
|
+
thread = Thread.new { with_context(context) { capture(job) } }
|
|
23
39
|
thread.report_on_exception = false
|
|
24
40
|
thread
|
|
25
41
|
end
|
|
@@ -28,123 +44,71 @@ module Squishling
|
|
|
28
44
|
threads&.each { |thread| thread.kill.join if thread.alive? }
|
|
29
45
|
end
|
|
30
46
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
[nil, e]
|
|
35
|
-
end
|
|
36
|
-
|
|
37
|
-
def self.instrumentation_context
|
|
38
|
-
instrumentation = defined?(RubyLLM::Support::Instrumentation) && RubyLLM::Support::Instrumentation
|
|
39
|
-
return unless instrumentation.respond_to?(:current_workflow) && instrumentation.respond_to?(:capture_context)
|
|
40
|
-
|
|
41
|
-
[instrumentation.current_workflow, instrumentation.capture_context]
|
|
42
|
-
end
|
|
43
|
-
|
|
44
|
-
def self.with_instrumentation_context(context, &)
|
|
45
|
-
return yield unless context
|
|
46
|
-
|
|
47
|
-
instrumentation = RubyLLM::Support::Instrumentation
|
|
48
|
-
instrumentation.with_workflow(context.first) { instrumentation.with_context(context.last, &) }
|
|
47
|
+
# A fresh conversation on one escalation step, with the system prompt and the strict output schema.
|
|
48
|
+
def chat(step, instructions:, schema:)
|
|
49
|
+
Conversation.new(step, instructions, schema, [])
|
|
49
50
|
end
|
|
50
51
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
52
|
+
# One request in the conversation: the message goes in, and the adapter's answer is kept in it so the next
|
|
53
|
+
# message continues from there. A request that fails leaves the conversation as it was.
|
|
54
|
+
# The adapter raises ConfigurationError for setup mistakes (never retried) and LLMError for failures worth another
|
|
55
|
+
# attempt, which moves on to the next attempt of the escalation (or propagates from the last one).
|
|
56
|
+
# The request runs inside the configured observer (see Observer), on this thread: for samples, the worker's.
|
|
57
|
+
# role is nil for an escalation attempt, :sample (with its name) or :judge; attempt counts in attempts.
|
|
58
|
+
# reask is whether this message is a retry in the same chat (see ModelPath::Step#request_params).
|
|
59
|
+
def ask(chat, message, step, attempt:, attempts:, role: nil, sample: nil, reask: false)
|
|
60
|
+
call = call_info(step, :chat, role:, sample:, attempt:, attempts:)
|
|
61
|
+
Observer.around(call) { complete(chat, message, step, reask) }
|
|
55
62
|
end
|
|
56
63
|
|
|
57
|
-
#
|
|
58
|
-
def
|
|
59
|
-
|
|
60
|
-
chat.with_instructions(instructions)
|
|
61
|
-
chat.with_schema(schema)
|
|
62
|
-
apply_params(chat, step.params)
|
|
63
|
-
chat
|
|
64
|
+
# Whether the adapter can make System One judgments (judges with type: :judgment).
|
|
65
|
+
def judgments?
|
|
66
|
+
@adapter.respond_to?(:judgments?) && @adapter.respond_to?(:judge) && @adapter.judgments?
|
|
64
67
|
end
|
|
65
68
|
|
|
66
|
-
#
|
|
67
|
-
#
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
# fallback would otherwise hide it on every call.
|
|
71
|
-
def ask(chat, message, step)
|
|
72
|
-
request(step) { chat.ask(message) }
|
|
69
|
+
# A System One judgment on one step, through the adapter.
|
|
70
|
+
# Judge#check_judgments! has already refused an adapter that can't.
|
|
71
|
+
def judge(input, questions:, step:)
|
|
72
|
+
@tally.add(@adapter.judge(input, questions:, step:, label: @label))
|
|
73
73
|
end
|
|
74
74
|
|
|
75
|
-
#
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
# runs inside the call) is the developer's own and propagates as-is.
|
|
80
|
-
def judge(input, questions:, step:)
|
|
81
|
-
request(step) do
|
|
82
|
-
RubyLLM.judge(input, questions:, provider_options: step.params, **model_options(step))
|
|
83
|
-
rescue RubyLLM::ModelNotFoundError => e
|
|
84
|
-
raise ConfigurationError, "#{@label}: #{e.message}"
|
|
85
|
-
rescue ArgumentError => e
|
|
86
|
-
raise unless raised_by_ruby_llm?(e)
|
|
87
|
-
|
|
88
|
-
raise ConfigurationError, "#{@label}: #{e.message}"
|
|
89
|
-
rescue RubyLLM::Error => e
|
|
90
|
-
raise unless e.instance_of?(RubyLLM::Error) && e.response.nil?
|
|
91
|
-
|
|
92
|
-
raise ConfigurationError, "#{@label}: #{e.message}"
|
|
93
|
-
end
|
|
75
|
+
# #judge inside the configured observer (see Observer), attempt counting in attempts.
|
|
76
|
+
def observed_judgment(input, questions:, step:, attempt:, attempts:)
|
|
77
|
+
call = call_info(step, :judgment, role: :judge, attempt:, attempts:)
|
|
78
|
+
Observer.around(call) { judge(input, questions:, step:) }
|
|
94
79
|
end
|
|
95
80
|
|
|
96
81
|
private
|
|
97
82
|
|
|
98
|
-
def
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
def build_chat(step)
|
|
103
|
-
RubyLLM.chat(**model_options(step))
|
|
104
|
-
rescue RubyLLM::ModelNotFoundError, RubyLLM::ConfigurationError => e
|
|
105
|
-
raise ConfigurationError, "#{@label}: #{e.message}"
|
|
83
|
+
def capture(job)
|
|
84
|
+
[job.call, nil]
|
|
85
|
+
rescue Exception => e # rubocop:disable Lint/RescueException -- re-raised on the caller's thread
|
|
86
|
+
[nil, e]
|
|
106
87
|
end
|
|
107
88
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
89
|
+
def complete(chat, message, step, reask)
|
|
90
|
+
messages = chat.messages + [{ role: :user, content: message }]
|
|
91
|
+
request = Adapter::Request.new(label: @label, model: step.model, provider: step.provider,
|
|
92
|
+
instructions: chat.instructions, messages:, schema: chat.schema, params: step.request_params(reask))
|
|
93
|
+
response = @adapter.complete(request)
|
|
94
|
+
unless response.is_a?(Adapter::Response)
|
|
95
|
+
raise ConfigurationError, "#{@label}: the #{@adapter.class} adapter must return a " \
|
|
96
|
+
"Squishling::Adapter::Response from #complete, got #{response.class}"
|
|
97
|
+
end
|
|
115
98
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
rescue RubyLLM::ConfigurationError, RubyLLM::UnauthorizedError, RubyLLM::ForbiddenError => e
|
|
119
|
-
raise ConfigurationError, "#{@label}: #{e.class}: #{e.message}"
|
|
120
|
-
rescue RubyLLM::BadRequestError => e
|
|
121
|
-
raise ConfigurationError,
|
|
122
|
-
"#{@label}: the provider rejected the request (#{e.message})#{params_hint(step.params)}"
|
|
123
|
-
rescue RubyLLM::Error, Faraday::Error => e
|
|
124
|
-
raise LLMError, "#{@label}: #{e.class}: #{e.message}"
|
|
99
|
+
chat.messages = messages + [{ role: :assistant, content: response.text }]
|
|
100
|
+
@tally.add(response)
|
|
125
101
|
end
|
|
126
102
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
def model_options(step)
|
|
130
|
-
model = step.model
|
|
131
|
-
provider = step.provider
|
|
132
|
-
options = { model:, provider: }.compact
|
|
133
|
-
options[:assume_model_exists] = true if model && provider && !known_model?(model, provider)
|
|
134
|
-
options
|
|
135
|
-
end
|
|
103
|
+
def with_context(context, &)
|
|
104
|
+
return yield unless @adapter.respond_to?(:with_context)
|
|
136
105
|
|
|
137
|
-
|
|
138
|
-
RubyLLM.models.find(model, provider:)
|
|
139
|
-
true
|
|
140
|
-
rescue RubyLLM::ModelNotFoundError
|
|
141
|
-
false
|
|
106
|
+
@adapter.with_context(context, &)
|
|
142
107
|
end
|
|
143
108
|
|
|
144
|
-
def
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
". Check params #{params.inspect}; reasoning models often reject sampling params such as temperature and top_p"
|
|
109
|
+
def call_info(step, request, attempt:, attempts:, role: nil, sample: nil)
|
|
110
|
+
Observer::Call.new(label: @label, purpose: @purpose, harness: @harness, handed_off: @handed_off, role:, sample:,
|
|
111
|
+
attempt:, attempts:, model: step.model, provider: step.provider, request:)
|
|
148
112
|
end
|
|
149
113
|
end
|
|
150
114
|
end
|
|
@@ -8,20 +8,35 @@ module Squishling
|
|
|
8
8
|
# "claude-sonnet-5-5",
|
|
9
9
|
# { model: "claude-opus-5-5", params: { thinking: { effort: :high } } }
|
|
10
10
|
# ]
|
|
11
|
-
# attempts: (default 1) retries a step;
|
|
12
|
-
#
|
|
13
|
-
# the
|
|
11
|
+
# attempts: (default 1) retries a step; priority: (all steps or none, lowest first) makes the order explicit
|
|
12
|
+
# instead of positional. Steps sharing a priority are one tier: each call rotates which of them goes first
|
|
13
|
+
# (see Rotation), and the others follow before the next tier, unless a step in the tier sets rotate: false.
|
|
14
|
+
# order: is the deprecated name of priority:. forward_rejected: false starts a step from the original input
|
|
15
|
+
# alone, without the previous step's rejected output. include_when: (a Proc, called like squish_when with the
|
|
16
|
+
# method's inputs as keywords, on the instance) drops the step from any call where it is falsy.
|
|
17
|
+
# retry_temperature: (a Numeric) is the temperature for the step's same-chat retries after the first attempt.
|
|
14
18
|
module ModelPath
|
|
15
|
-
# One attempt: the model, its provider, the fully resolved generation params,
|
|
16
|
-
# step's rejected output is shown to it.
|
|
17
|
-
Step = Data.define(:model, :provider, :params, :forward_rejected) do
|
|
18
|
-
|
|
19
|
+
# One attempt: the model, its provider, the fully resolved generation params, whether the previous
|
|
20
|
+
# step's rejected output is shown to it, and the temperature its retries in the same chat use, if any.
|
|
21
|
+
Step = Data.define(:model, :provider, :params, :forward_rejected, :retry_temperature) do
|
|
22
|
+
def initialize(model:, provider:, params:, forward_rejected:, retry_temperature: nil)
|
|
23
|
+
super
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# The params for a request: on a retry in the same chat (reask), the step's retry_temperature replaces the
|
|
27
|
+
# temperature, if it sets one.
|
|
28
|
+
def request_params(reask)
|
|
29
|
+
reask && retry_temperature ? Params.resolve(params, temperature: retry_temperature) : params
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# The model for logs and messages (nil means the adapter's default model).
|
|
19
33
|
def display_name
|
|
20
|
-
model || "
|
|
34
|
+
model || "the adapter's default model"
|
|
21
35
|
end
|
|
22
36
|
end
|
|
23
37
|
|
|
24
|
-
STEP_KEYS = %i[model provider params attempts order forward_rejected
|
|
38
|
+
STEP_KEYS = %i[model provider params attempts priority order rotate forward_rejected include_when
|
|
39
|
+
retry_temperature].freeze
|
|
25
40
|
|
|
26
41
|
module_function
|
|
27
42
|
|
|
@@ -57,43 +72,75 @@ module Squishling
|
|
|
57
72
|
end
|
|
58
73
|
|
|
59
74
|
# One Step per attempt. The level's provider applies to steps that don't name their own, and each
|
|
60
|
-
# step's params are merged over the level-resolved params.
|
|
61
|
-
|
|
62
|
-
|
|
75
|
+
# step's params are merged over the level-resolved params. turn is this call's place in the method's rotation
|
|
76
|
+
# (see Rotation): each tier of tied priorities starts turn steps further along.
|
|
77
|
+
def steps(path, provider:, params:, turn: 0)
|
|
78
|
+
rotated(path, turn).flat_map do |step|
|
|
63
79
|
attempt = Step.new(model: step[:model], provider: step[:provider] || provider,
|
|
64
|
-
params: Params.resolve(params, step[:params]), forward_rejected: step[:forward_rejected]
|
|
80
|
+
params: Params.resolve(params, step[:params]), forward_rejected: step[:forward_rejected],
|
|
81
|
+
retry_temperature: step[:retry_temperature])
|
|
65
82
|
[attempt] * step[:attempts]
|
|
66
83
|
end
|
|
67
84
|
end
|
|
68
85
|
|
|
86
|
+
# The steps this call may use: those without an include_when:, and those whose include_when: is truthy for the
|
|
87
|
+
# call's inputs (evaluated against the receiver, like squish_when). Priority order is kept.
|
|
88
|
+
def eligible(path, receiver, inputs, label)
|
|
89
|
+
kept = path.select do |step|
|
|
90
|
+
gate = step[:include_when]
|
|
91
|
+
gate.nil? || receiver.instance_exec(**inputs, &gate)
|
|
92
|
+
end
|
|
93
|
+
raise ConfigurationError, "#{label}: include_when: excluded every escalation step" if kept.empty?
|
|
94
|
+
|
|
95
|
+
kept
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Whether any tier of the path has several models to rotate between.
|
|
99
|
+
def rotating?(path)
|
|
100
|
+
tiers(path).any? { |tier| rotates?(tier) }
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Steps sorted by priority, lowest first, ties kept in the order declared.
|
|
69
104
|
def ordered(steps, label)
|
|
70
|
-
numbered = steps.count { |step| step.key?(:
|
|
71
|
-
return steps
|
|
105
|
+
numbered = steps.count { |step| step.key?(:priority) }
|
|
106
|
+
return steps if numbered.zero?
|
|
72
107
|
|
|
73
108
|
if numbered < steps.size
|
|
74
|
-
raise ConfigurationError,
|
|
109
|
+
raise ConfigurationError,
|
|
110
|
+
"#{label}: give every step a priority: or none (#{numbered} of #{steps.size} have one)"
|
|
75
111
|
end
|
|
76
112
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
113
|
+
steps.each_with_index.sort_by { |step, index| [step[:priority], index] }.map(&:first)
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Runs of steps with the same priority. A step without one is a tier of its own.
|
|
117
|
+
def tiers(path)
|
|
118
|
+
path.slice_when { |before, after| before[:priority].nil? || before[:priority] != after[:priority] }
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def rotates?(tier)
|
|
122
|
+
tier.size > 1 && tier.none? { |step| step[:rotate] == false }
|
|
123
|
+
end
|
|
80
124
|
|
|
81
|
-
|
|
125
|
+
def rotated(path, turn)
|
|
126
|
+
tiers(path).flat_map { |tier| rotates?(tier) ? tier.rotate(turn % tier.size) : tier }
|
|
82
127
|
end
|
|
83
128
|
|
|
84
|
-
|
|
129
|
+
# extras: names more keys a step Hash may carry for its caller (a judge's type:, say). They are kept as given,
|
|
130
|
+
# for the caller to check.
|
|
131
|
+
def normalize_step(step, label, extras: [])
|
|
85
132
|
case step
|
|
86
133
|
when String, Symbol
|
|
87
134
|
{ model: model_name(step, label), provider: nil, params: nil, attempts: 1, forward_rejected: true }
|
|
88
|
-
when Hash then normalize_hash(step, label)
|
|
135
|
+
when Hash then normalize_hash(step, label, extras)
|
|
89
136
|
else
|
|
90
137
|
raise ConfigurationError, "#{label}: each step must be a model name or a Hash with model:, got #{step.inspect}"
|
|
91
138
|
end
|
|
92
139
|
end
|
|
93
140
|
|
|
94
|
-
def normalize_hash(step, label)
|
|
141
|
+
def normalize_hash(step, label, extras)
|
|
95
142
|
step = step.transform_keys(&:to_sym)
|
|
96
|
-
unknown = step.keys - STEP_KEYS
|
|
143
|
+
unknown = step.keys - STEP_KEYS - extras
|
|
97
144
|
raise ConfigurationError, "#{label}: unknown step option(s) #{unknown.join(', ')}" if unknown.any?
|
|
98
145
|
unless step[:model].is_a?(String) || step[:model].is_a?(Symbol)
|
|
99
146
|
raise ConfigurationError, "#{label}: a step needs a model: name, got #{step.inspect}"
|
|
@@ -104,10 +151,17 @@ module Squishling
|
|
|
104
151
|
unless attempts.is_a?(Integer) && attempts.positive?
|
|
105
152
|
raise ConfigurationError, "#{label} (#{model}): attempts: must be a positive Integer, got #{attempts.inspect}"
|
|
106
153
|
end
|
|
107
|
-
|
|
108
|
-
|
|
154
|
+
|
|
155
|
+
step = priority_alias(step, label, model)
|
|
156
|
+
if step.key?(:priority) && !step[:priority].is_a?(Integer)
|
|
157
|
+
raise ConfigurationError, "#{label} (#{model}): priority: must be an Integer, got #{step[:priority].inspect}"
|
|
158
|
+
end
|
|
159
|
+
if step.key?(:rotate) && ![true, false].include?(step[:rotate])
|
|
160
|
+
raise ConfigurationError, "#{label} (#{model}): rotate: must be true or false, got #{step[:rotate].inspect}"
|
|
109
161
|
end
|
|
110
162
|
|
|
163
|
+
check_include_when!(step, label, model)
|
|
164
|
+
check_retry_temperature!(step, label, model)
|
|
111
165
|
forward_rejected = step.fetch(:forward_rejected, true)
|
|
112
166
|
unless [true, false].include?(forward_rejected)
|
|
113
167
|
raise ConfigurationError,
|
|
@@ -116,7 +170,43 @@ module Squishling
|
|
|
116
170
|
|
|
117
171
|
params = step[:params] && Params.normalize(step[:params], "#{label} (#{model}) params")
|
|
118
172
|
normalized = { model:, provider: step[:provider], params:, attempts:, forward_rejected: }
|
|
119
|
-
step.
|
|
173
|
+
normalized.merge(step.slice(:priority, :rotate, :include_when, :retry_temperature, *extras))
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def check_include_when!(step, label, model)
|
|
177
|
+
return if !step.key?(:include_when) || step[:include_when].is_a?(Proc)
|
|
178
|
+
|
|
179
|
+
raise ConfigurationError,
|
|
180
|
+
"#{label} (#{model}): include_when: must be a Proc, got #{step[:include_when].inspect}"
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def valid_temperature?(temperature)
|
|
184
|
+
temperature.is_a?(Numeric) && temperature.real? && temperature.finite? && !temperature.negative?
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
def check_retry_temperature!(step, label, model)
|
|
188
|
+
temperature = step[:retry_temperature]
|
|
189
|
+
return if !step.key?(:retry_temperature) || valid_temperature?(temperature)
|
|
190
|
+
|
|
191
|
+
raise ConfigurationError,
|
|
192
|
+
"#{label} (#{model}): retry_temperature: must be a finite Numeric of 0 or more, got #{temperature.inspect}"
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# A step used on its own (a judge, a harness's step) can't take part in a tier or be left out of a call, so it
|
|
196
|
+
# takes no priority:, rotate:, or include_when:. who names it for the message.
|
|
197
|
+
def single_step!(step, who)
|
|
198
|
+
return unless step.key?(:priority) || step.key?(:rotate) || step.key?(:include_when)
|
|
199
|
+
|
|
200
|
+
raise ConfigurationError, "#{who} takes no priority:, rotate:, or include_when:"
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
# order: is the deprecated name of priority:.
|
|
204
|
+
def priority_alias(step, label, model)
|
|
205
|
+
return step unless step.key?(:order)
|
|
206
|
+
raise ConfigurationError, "#{label} (#{model}): give priority: or order:, not both" if step.key?(:priority)
|
|
207
|
+
|
|
208
|
+
warn "[Squishling] #{label} (#{model}): order: is deprecated, use priority:"
|
|
209
|
+
step.except(:order).merge(priority: step[:order])
|
|
120
210
|
end
|
|
121
211
|
|
|
122
212
|
def model_name(model, label)
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Squishling
|
|
4
|
+
# Classes by name, for the things that can be registered (harnesses, adapters): the built-ins, registered as they
|
|
5
|
+
# load, and the developer's own. Shared across threads, so every read and write is under the lock.
|
|
6
|
+
class NamedRegistry
|
|
7
|
+
NAME = /\A[a-z][a-z0-9_]*\z/
|
|
8
|
+
|
|
9
|
+
# kind names what is registered ("harness"), for messages.
|
|
10
|
+
def initialize(kind)
|
|
11
|
+
@kind = kind
|
|
12
|
+
@lock = Mutex.new
|
|
13
|
+
@built_in = {}
|
|
14
|
+
@custom = {}
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def register_built_in(name, klass)
|
|
18
|
+
@lock.synchronize { @built_in[name] = klass }
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# The name as a Symbol. Raises ConfigurationError when it isn't a Symbol or String of lowercase letters, digits
|
|
22
|
+
# and underscores, starting with a letter.
|
|
23
|
+
def checked_name(name)
|
|
24
|
+
unless name.is_a?(Symbol) || name.is_a?(String)
|
|
25
|
+
raise ConfigurationError, "#{@kind} name must be a Symbol or String, got #{name.inspect}"
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
name = name.to_sym
|
|
29
|
+
return name if NAME.match?(name)
|
|
30
|
+
|
|
31
|
+
raise ConfigurationError,
|
|
32
|
+
"#{@kind} name #{name.inspect} must be lowercase letters, digits, and underscores, starting with a letter"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# Registers klass under name (already checked), unless the name is taken.
|
|
36
|
+
def add(name, klass)
|
|
37
|
+
@lock.synchronize do
|
|
38
|
+
raise ConfigurationError, "#{@kind} #{name.inspect} is already registered" if taken?(name)
|
|
39
|
+
|
|
40
|
+
@custom[name] = klass
|
|
41
|
+
end
|
|
42
|
+
klass
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def fetch(name)
|
|
46
|
+
@lock.synchronize { @built_in[name] || @custom[name] }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def names
|
|
50
|
+
@lock.synchronize { @built_in.keys + @custom.keys }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Forgets the registered classes; the built-ins stay.
|
|
54
|
+
def reset!
|
|
55
|
+
@lock.synchronize { @custom.clear }
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
private
|
|
59
|
+
|
|
60
|
+
def taken?(name)
|
|
61
|
+
@built_in.key?(name) || @custom.key?(name)
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Squishling
|
|
4
|
+
# The around-style hook for tools that must see an LLM request while it is made (a tracing span, a Coolhand
|
|
5
|
+
# declared workload), which squawk's after-the-fact report can't do. An observer is any object responding to
|
|
6
|
+
# around(call), which yields once to make the request. See docs/observers.md.
|
|
7
|
+
#
|
|
8
|
+
# Observers get call metadata (Call) and, back from the yield, the request's token Usage: never the model's
|
|
9
|
+
# output or the method's inputs.
|
|
10
|
+
module Observer
|
|
11
|
+
# What an observer is told about one provider request. handed_off is true when squish! sent the call to the
|
|
12
|
+
# LLM (typically from a rescue) and false when squish_when or a missing implementation did. role is nil for an
|
|
13
|
+
# escalation attempt, :sample (with sample "a", "b", ...) or :judge, like squawk's metadata. attempt and
|
|
14
|
+
# attempts count the (expanded) escalation path the request belongs to. request is :chat, or :judgment for a
|
|
15
|
+
# System One judge.
|
|
16
|
+
Call = Data.define(:label, :purpose, :harness, :handed_off, :role, :sample, :attempt, :attempts, :model, :provider,
|
|
17
|
+
:request)
|
|
18
|
+
|
|
19
|
+
# What an observer's block returns: the request's token usage and the model that answered, never the content.
|
|
20
|
+
# A count the provider didn't report is nil.
|
|
21
|
+
Usage = Data.define(:input_tokens, :output_tokens, :model)
|
|
22
|
+
|
|
23
|
+
BUILT_IN = %i[coolhand].freeze
|
|
24
|
+
|
|
25
|
+
module_function
|
|
26
|
+
|
|
27
|
+
# The observer itself, nil (auto-detect), or false (none). A built-in's name (:coolhand) or anything that
|
|
28
|
+
# responds to around is accepted, checked here so a mistake fails at setup, not inside the first request.
|
|
29
|
+
def validate(observer, where)
|
|
30
|
+
return observer if observer.nil? || observer == false
|
|
31
|
+
|
|
32
|
+
if observer.is_a?(Symbol)
|
|
33
|
+
return observer if BUILT_IN.include?(observer)
|
|
34
|
+
|
|
35
|
+
raise ConfigurationError, "#{where} observer must be one of #{BUILT_IN.map(&:inspect).join(', ')}, " \
|
|
36
|
+
"an object that responds to around, or false, got #{observer.inspect}"
|
|
37
|
+
end
|
|
38
|
+
return observer if observer.respond_to?(:around)
|
|
39
|
+
|
|
40
|
+
raise ConfigurationError, "#{where} observer must respond to around(call) { ... } (or be false), " \
|
|
41
|
+
"got #{observer.inspect}"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# The observer that wraps a request: the configured one, the Coolhand observer when nothing is configured and
|
|
45
|
+
# Coolhand is loaded and set up, or nil.
|
|
46
|
+
def resolve(configured)
|
|
47
|
+
case configured
|
|
48
|
+
when false then nil
|
|
49
|
+
when nil then Observers::Coolhand.new if Observers::Coolhand.available?
|
|
50
|
+
when Symbol then Observers::Coolhand.new
|
|
51
|
+
else configured
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Runs the request inside the resolved observer, if there is one. The block runs on the calling thread, so
|
|
56
|
+
# for samples it is the worker thread that makes the request. The observer gets the request's Usage back from
|
|
57
|
+
# its yield, not the response, so the model's output can't leave the process through it, and what it returns
|
|
58
|
+
# is ignored, and so is any rescue of the request's error: the error is raised again. An observer that never
|
|
59
|
+
# yields would make the request vanish, and one that yields twice would send it twice: both are setup mistakes.
|
|
60
|
+
def around(call)
|
|
61
|
+
observer = resolve(Squishling.config.observer)
|
|
62
|
+
return yield unless observer
|
|
63
|
+
|
|
64
|
+
yielded = false
|
|
65
|
+
response = nil
|
|
66
|
+
failure = nil
|
|
67
|
+
observer.around(call) do
|
|
68
|
+
if yielded
|
|
69
|
+
failure = ConfigurationError.new("#{call.label}: #{observer.class}#around must yield only once")
|
|
70
|
+
raise failure
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
yielded = true
|
|
74
|
+
begin
|
|
75
|
+
response = yield
|
|
76
|
+
rescue StandardError => e
|
|
77
|
+
failure = e
|
|
78
|
+
raise
|
|
79
|
+
end
|
|
80
|
+
usage(response)
|
|
81
|
+
end
|
|
82
|
+
unless yielded
|
|
83
|
+
raise ConfigurationError,
|
|
84
|
+
"#{call.label}: the #{observer.class} observer's around must yield to make the request"
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
raise failure if failure # an observer can't swallow the request's error: Squishling's taxonomy holds
|
|
88
|
+
|
|
89
|
+
response
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# An Adapter::Response carries its token counts in a Hash; a judgment (see Adapter::Base#judge) may carry them in
|
|
93
|
+
# an object with input and output instead. Shared with Budget::Tally; not part of the public API.
|
|
94
|
+
def usage(response)
|
|
95
|
+
tokens = response.respond_to?(:tokens) ? response.tokens : nil
|
|
96
|
+
model = response.respond_to?(:model) ? response.model : nil
|
|
97
|
+
Usage.new(input_tokens: token_count(tokens, :input_tokens, :input),
|
|
98
|
+
output_tokens: token_count(tokens, :output_tokens, :output),
|
|
99
|
+
model: model.respond_to?(:id) ? model.id : model)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def token_count(tokens, key, reader)
|
|
103
|
+
return tokens[key] if tokens.is_a?(Hash)
|
|
104
|
+
|
|
105
|
+
tokens.respond_to?(reader) ? tokens.public_send(reader) : nil
|
|
106
|
+
end
|
|
107
|
+
private_class_method :token_count
|
|
108
|
+
end
|
|
109
|
+
end
|