squishling 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +133 -1
- data/README.md +63 -15
- data/docs/adapters.md +189 -0
- data/docs/configuration.md +89 -24
- data/docs/coolhand.md +66 -0
- data/docs/failures.md +129 -20
- data/docs/harnesses.md +186 -48
- data/docs/measuring-tokens.md +36 -42
- data/docs/observers.md +80 -0
- data/docs/routing.md +7 -2
- data/docs/schemas.md +33 -6
- data/docs/testing.md +183 -0
- data/docs/verifying.md +84 -0
- data/lib/squishling/adapter/error_body.rb +47 -0
- data/lib/squishling/adapter.rb +275 -0
- data/lib/squishling/adapters/openai_compatible.rb +304 -0
- data/lib/squishling/adapters/openai_options.rb +72 -0
- data/lib/squishling/adapters/ruby_llm.rb +193 -0
- data/lib/squishling/budget.rb +71 -0
- data/lib/squishling/class_methods.rb +43 -7
- data/lib/squishling/class_registry.rb +63 -0
- data/lib/squishling/configuration.rb +72 -8
- data/lib/squishling/definition.rb +95 -17
- data/lib/squishling/errors.rb +97 -12
- data/lib/squishling/harness/base.rb +136 -0
- data/lib/squishling/harness/escalation.rb +22 -0
- data/lib/squishling/harness/judged_squishsum.rb +83 -0
- data/lib/squishling/harness/option_types.rb +94 -0
- data/lib/squishling/harness/registry.rb +41 -0
- data/lib/squishling/harness/squishsum.rb +22 -0
- data/lib/squishling/harness.rb +25 -99
- data/lib/squishling/invoker.rb +28 -233
- data/lib/squishling/judge.rb +78 -21
- data/lib/squishling/llm_client.rb +73 -109
- data/lib/squishling/model_path.rb +117 -27
- data/lib/squishling/named_registry.rb +64 -0
- data/lib/squishling/observer.rb +109 -0
- data/lib/squishling/observers/coolhand.rb +86 -0
- data/lib/squishling/output_check.rb +13 -9
- data/lib/squishling/params.rb +15 -27
- data/lib/squishling/railtie.rb +10 -0
- data/lib/squishling/result.rb +37 -16
- data/lib/squishling/rotation.rb +31 -0
- data/lib/squishling/rspec.rb +18 -0
- data/lib/squishling/run/messages.rb +79 -0
- data/lib/squishling/run/rejections.rb +43 -0
- data/lib/squishling/run.rb +320 -0
- data/lib/squishling/schema.rb +10 -9
- data/lib/squishling/squawk.rb +33 -3
- data/lib/squishling/tasks/squishling.rake +18 -0
- data/lib/squishling/testing/adapter.rb +194 -0
- data/lib/squishling/testing/checks.rb +39 -0
- data/lib/squishling/testing/direct.rb +41 -0
- data/lib/squishling/testing/errors.rb +18 -0
- data/lib/squishling/testing/fixtures.rb +206 -0
- data/lib/squishling/testing/helpers.rb +71 -0
- data/lib/squishling/testing/minitest.rb +32 -0
- data/lib/squishling/testing/observer.rb +31 -0
- data/lib/squishling/testing/reply.rb +60 -0
- data/lib/squishling/testing/request.rb +75 -0
- data/lib/squishling/testing/rspec.rb +30 -0
- data/lib/squishling/testing/script.rb +189 -0
- data/lib/squishling/testing/state.rb +73 -0
- data/lib/squishling/testing/stubs.rb +112 -0
- data/lib/squishling/testing.rb +126 -0
- data/lib/squishling/verification.rb +39 -0
- data/lib/squishling/verifier.rb +179 -0
- data/lib/squishling/version.rb +1 -1
- data/lib/squishling.rb +89 -4
- metadata +53 -21
data/docs/testing.md
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# Testing
|
|
2
|
+
|
|
3
|
+
`require "squishling/testing"` adds test support: a scripted [adapter](adapters.md) that answers in place of a
|
|
4
|
+
provider and records what was sent to it, stand-ins for the callers of squished methods, replies derived from an output
|
|
5
|
+
schema, and assertions. It is opt-in (`require "squishling"` doesn't load it), needs no test framework or mocking
|
|
6
|
+
library, and works with RSpec (without rspec-mocks), Minitest, and Mocha alike. It makes no network request.
|
|
7
|
+
|
|
8
|
+
```ruby
|
|
9
|
+
# Minitest
|
|
10
|
+
include Squishling::Testing::Minitest
|
|
11
|
+
|
|
12
|
+
# RSpec
|
|
13
|
+
RSpec.configure { |config| config.include Squishling::Testing::RSpec }
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
Both give your tests the helpers below and put back, after each test, whatever the helpers changed
|
|
17
|
+
(`Squishling::Testing.reset!`). They touch only the adapter, observer, and stubs they installed (and only while they
|
|
18
|
+
still hold what the helpers set), so the rest of your `Squishling.configure` survives.
|
|
19
|
+
|
|
20
|
+
The helpers change global state (`Squishling.config` and the stubbed classes), so tests that use them must not run
|
|
21
|
+
in parallel threads.
|
|
22
|
+
|
|
23
|
+
## Scripting the model
|
|
24
|
+
|
|
25
|
+
```ruby
|
|
26
|
+
fake = fake_squishling do |llm|
|
|
27
|
+
llm.step("gemini-2.5-flash").reply({ regex: "[bad(" }, { regex: "Run.*" }) # in order; the last one repeats
|
|
28
|
+
llm.step("gemini-2.5-pro").fail(:rate_limit)
|
|
29
|
+
llm.step(:any).reply({ regex: "fallback.*" }) # anything no rule above claims
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
result = RegexInferrer.call(sample_turns: turns)
|
|
33
|
+
|
|
34
|
+
assert_equal "Run.*", result.regex
|
|
35
|
+
assert_equal 2, fake.requests.size
|
|
36
|
+
assert_sent_arguments fake, { sample_turns: turns } # what left the process
|
|
37
|
+
assert_includes fake.requests.last.errors, "regex is not a valid Ruby regex" # the re-ask carried them
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
A reply is one of:
|
|
41
|
+
|
|
42
|
+
| Reply | The model… |
|
|
43
|
+
|---|---|
|
|
44
|
+
| a `Hash` | answered with this object |
|
|
45
|
+
| a `String` | returned this text (malformed or fenced JSON, truncated output) |
|
|
46
|
+
| `nil` | returned nothing (a refusal or a cut-off) |
|
|
47
|
+
| `Squishling::Testing::Reply.new(content, model: "m", usage: { input_tokens: 10, output_tokens: 3 })` | answered, and the provider reported this model and these token counts (`squawk`'s `metadata[:usage]`, the observer's `Usage`) |
|
|
48
|
+
| an `Exception` | made the request raise it |
|
|
49
|
+
| `.fail(intent)` | the provider failed: see below |
|
|
50
|
+
|
|
51
|
+
`reply` takes several, `.fail` adds one, and `.once` makes running out of replies an error instead of repeating the last.
|
|
52
|
+
|
|
53
|
+
**Provider failures by intent.** You don't need to know which SDK error maps to which Squishling error:
|
|
54
|
+
|
|
55
|
+
| Intent | Raises |
|
|
56
|
+
|---|---|
|
|
57
|
+
| `:rate_limit`, `:server_error`, `:overloaded`, `:timeout`, `:context_length` | `LLMError` (the next attempt is tried) |
|
|
58
|
+
| `:bad_request`, `:unauthorized`, `:forbidden` | `ConfigurationError` (never retried, never given to a fallback) |
|
|
59
|
+
|
|
60
|
+
**Selecting what a rule answers.** `step(model)`, `step(provider: :vertexai)`, and `step(:any)` answer escalation
|
|
61
|
+
attempts; `attempt: 2` narrows a step rule to one attempt of the escalation. Rules are matched most specific first
|
|
62
|
+
(a sample or the judge, then the rule naming the most of model, provider and attempt, then `step(:any)`), and by
|
|
63
|
+
declaration order within that.
|
|
64
|
+
Scripts are keyed by model, so a rotated tier (`priority:`) gets the same answers whichever model goes first. Which
|
|
65
|
+
model goes first still depends on how many calls the class has had, so don't assert the order of `fake.models` for a
|
|
66
|
+
rotated tier unless the class is new to the test (or pin the tier with `rotate: false`).
|
|
67
|
+
A request no rule covers raises `Squishling::Testing::UnscriptedRequestError` naming the squished method; it isn't
|
|
68
|
+
a `Squishling::Error`, so `squish_fallback` can't hide it.
|
|
69
|
+
|
|
70
|
+
### Harnesses
|
|
71
|
+
|
|
72
|
+
```ruby
|
|
73
|
+
fake_squishling do |llm|
|
|
74
|
+
llm.sample("a").reply(valid_reply_for(Triager))
|
|
75
|
+
llm.sample("b").reply(valid_reply_for(Triager, team: "billing"))
|
|
76
|
+
llm.judge.verdict(:a, reason: "a fits") # a chat judge
|
|
77
|
+
llm.judge.judgment(:b, confidence: 0.92) # a System One judge (type: :judgment, min_confidence:)
|
|
78
|
+
llm.judge.fail_judgment(:timeout)
|
|
79
|
+
end
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
Each sample has its own script, so the order its worker thread happens to run in can't change what it gets. The
|
|
83
|
+
scripts, the request log, and the conversation tracking are taken under one lock. `DisagreementError#candidates`
|
|
84
|
+
and `#verdict` are the real ones; rescue them as you would in production.
|
|
85
|
+
|
|
86
|
+
## Inspecting what was sent
|
|
87
|
+
|
|
88
|
+
`fake.requests` is every provider request, in order, as `Squishling::Testing::Request`:
|
|
89
|
+
|
|
90
|
+
| | |
|
|
91
|
+
|---|---|
|
|
92
|
+
| `arguments`, `context` | the method's arguments and the `squish_context` / `squish!` context the model was sent |
|
|
93
|
+
| `system_prompt`, `output_schema` | the purpose (with `append_to_purpose` sections) and the strict schema |
|
|
94
|
+
| `model`, `provider`, `params` | the step it went to |
|
|
95
|
+
| `role`, `sample`, `attempt`, `attempts` | `nil`/`:sample`/`:judge`, `"a"`, and where it is in the escalation |
|
|
96
|
+
| `conversation` | the chat it belongs to |
|
|
97
|
+
| `errors` | the validation messages a retry or escalation carried back to the model |
|
|
98
|
+
| `forwarded_rejected` | the rejected output forwarded to the next step (`nil` with `forward_rejected: false`) |
|
|
99
|
+
| `candidates` | for a judge, the two samples |
|
|
100
|
+
| `messages`, `reply`, `failed?` | the conversation as sent, and the `Adapter::Response` (or error) that came back |
|
|
101
|
+
|
|
102
|
+
`fake.conversations` groups the requests by chat: a retry on the same step shares one, and an escalation to a new
|
|
103
|
+
step starts a new one. `fake.requests_for(sample: "a")`, `fake.models`, and `fake.arguments` are shortcuts.
|
|
104
|
+
|
|
105
|
+
## Asserting the LLM was not called
|
|
106
|
+
|
|
107
|
+
```ruby
|
|
108
|
+
assert_no_llm_call { RegexInferrer.call(sample_turns: easy_turns) } # then: refute result.squished?
|
|
109
|
+
assert_llm_calls fake, 2
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
`assert_no_llm_call` fails even if the code under test rescued the error that stopped the call.
|
|
113
|
+
|
|
114
|
+
**A safety net for the whole suite.** In a suite-wide `before` hook, `forbid_provider_calls!` makes any request that
|
|
115
|
+
reaches the provider raise `ProviderCallError` naming the squished method (`"RegexInferrer#call reached the provider…"`),
|
|
116
|
+
so a missing script fails at the call, not far from it. Installing a fake replaces it.
|
|
117
|
+
|
|
118
|
+
## Stubbing a squished class for its callers
|
|
119
|
+
|
|
120
|
+
Callers of a squished class shouldn't run Squishling at all. These make `Klass.call` return a real, schema-validated
|
|
121
|
+
typed result, or raise the real error class, without `allow(...).to receive`:
|
|
122
|
+
|
|
123
|
+
```ruby
|
|
124
|
+
stub_squishling_result(PatternGenerator, regex: "x") # squished? is true
|
|
125
|
+
stub_squishling_result(PatternGenerator, { regex: "x" }, squished: false) # as the Ruby path would return it
|
|
126
|
+
stub_squishling_result(PatternGenerator, regex: "x") { ... } # for the block only
|
|
127
|
+
|
|
128
|
+
stub_squishling_error(PatternGenerator, :invalid_output, errors: ["bad"], raw: "{}", attempts: 2, models: %w[a b])
|
|
129
|
+
stub_squishling_error(PatternGenerator, :llm_error) # also :configuration, :disagreement (candidates:, verdict:, reason:)
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Attributes that don't satisfy the method's `output_schema` raise `Squishling::Testing::SetupError` when you stub, so a
|
|
133
|
+
stand-in can't drift from the real result type. Pass `method: :triage` for another squished method. The stubs are
|
|
134
|
+
removed after each test. A stub applies to that class only, not to its subclasses, which keep their own behavior. The
|
|
135
|
+
hook that makes a stub possible stays on the class for the life of the process, so stub the classes of your suite, not
|
|
136
|
+
one per test.
|
|
137
|
+
|
|
138
|
+
## Running parts of a method without a model
|
|
139
|
+
|
|
140
|
+
```ruby
|
|
141
|
+
squishling_validate(Klass, { "regex" => "[bad(" }, text: "x") # => ["regex is not a valid Ruby regex"]
|
|
142
|
+
squishling_fallback_for(Klass, Squishling::LLMError.new("x"), text: "x") # the typed fallback result
|
|
143
|
+
squishling_preview(Klass, text: "x")
|
|
144
|
+
# => { system_prompt:, user_message:, output_schema:, step: } # for snapshot tests; makes no request
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
Arguments are keywords, or `args: [...]` for positional ones; `receiver:` supplies an instance for a class whose
|
|
148
|
+
constructor takes arguments; `method:` names a squished method other than `call`. (A squished method with a parameter
|
|
149
|
+
called `args`, `receiver`, or `method` can't be given it as a keyword here.) `squishling_preview` doesn't advance model rotation.
|
|
150
|
+
|
|
151
|
+
## Replies derived from the schema
|
|
152
|
+
|
|
153
|
+
```ruby
|
|
154
|
+
valid_reply_for(Triager) # => { "priority" => "example", "team" => "example" }
|
|
155
|
+
valid_reply_for(Triager, team: "billing") # overrides
|
|
156
|
+
invalid_reply_for(Triager, violating: :score) # out of its range, enum, or type
|
|
157
|
+
invalid_reply_for(Triager, violating: :score, how: :missing) # :enum, :range, :type, :null, :missing
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Both accept a squished class, a `Squishling::Schema`, or a schema Hash, and are checked against the schema before they
|
|
161
|
+
are returned, so a test that doesn't care about the content stays valid when the schema changes. A schema it can't
|
|
162
|
+
satisfy on its own (a `pattern`, an unsatisfiable conditional) raises `SetupError` and asks for overrides.
|
|
163
|
+
|
|
164
|
+
## `squawk` and observers
|
|
165
|
+
|
|
166
|
+
```ruby
|
|
167
|
+
capture = with_squawk(capture_squawk) { Klass.call(text: "x") }
|
|
168
|
+
capture.calls.map { |call| call.metadata[:attempt] } # output:, error:, and metadata: of every attempt
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
`with_squawk` swaps `config.squawk` for the block and puts the previous hook back, so an app-wide hook that enqueues
|
|
172
|
+
a job isn't run in tests. A `squawk:` set on a class or method still takes precedence over the configured one.
|
|
173
|
+
|
|
174
|
+
An observer you configured *before* `fake_squishling` keeps working while the fake is installed (it is chained behind
|
|
175
|
+
the fake's own). Configure it first: one assigned afterwards replaces the fake's, and then sample and judge scripts
|
|
176
|
+
can't tell which request is which.
|
|
177
|
+
|
|
178
|
+
## How it works
|
|
179
|
+
|
|
180
|
+
`fake_squishling` assigns a `Squishling::Testing::Adapter` (a `Squishling::Adapter::Base`) as `config.adapter`, and an
|
|
181
|
+
observer that tells it which role, sample, and attempt each request is. Because it sits behind the same interface as
|
|
182
|
+
the real adapters, it needs no knowledge of RubyLLM or any provider, and it is covered by Squishling's own specs.
|
|
183
|
+
Call `Squishling::Testing.install` and `.reset!` yourself if you use a test framework other than the two above.
|
data/docs/verifying.md
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# Verifying squished classes
|
|
2
|
+
|
|
3
|
+
Most mistakes in a squished class fail when the class loads (a bad `escalation:`, `params:`, `harness:` or `squawk:`
|
|
4
|
+
raises `ConfigurationError` as the declaration runs). The output schema and its typed result are built on the first
|
|
5
|
+
call, and the keywords your `include_when:` and `squish_when` procs take are only used when a call is routed. So
|
|
6
|
+
Squishling keeps a list of the classes that include it and can check them all without making a request, for example
|
|
7
|
+
in CI.
|
|
8
|
+
|
|
9
|
+
## `Squishling.verify` and `verify!`
|
|
10
|
+
|
|
11
|
+
```ruby
|
|
12
|
+
report = Squishling.verify # every squished class
|
|
13
|
+
report.ok? # => false
|
|
14
|
+
puts report
|
|
15
|
+
# Squishling found 2 problems:
|
|
16
|
+
# Triage#route: escalation step "claude-haiku-4-5" include_when: requires tier:, which is not an input of the method
|
|
17
|
+
# Invoice#call: invalid data member: total=
|
|
18
|
+
|
|
19
|
+
Squishling.verify! # raises Squishling::VerificationError (a ConfigurationError) naming every problem
|
|
20
|
+
Squishling.verify(Triage, Invoice) # just these classes
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
`verify` returns a `Squishling::Verification::Report` (`ok?`, `size`, `problems`, `to_s`). Each problem has a
|
|
24
|
+
`label` and a `message`. `verify!` returns the report when it is clean and otherwise raises
|
|
25
|
+
`Squishling::VerificationError`, which has `report` and `problems`. Every problem is listed, not just the first.
|
|
26
|
+
|
|
27
|
+
For each squished method (inherited ones too, with the subclass's settings) it checks:
|
|
28
|
+
|
|
29
|
+
- **The output schema:** that it builds (and is strict), is valid JSON Schema, and produces a typed result, so a
|
|
30
|
+
property name that can't be a result member (`total=`) is reported. A method with no output schema or no purpose is reported, because its LLM path
|
|
31
|
+
would raise. The default `call` method is skipped when you don't implement or configure it.
|
|
32
|
+
- **What the layers resolve to:** the escalation, generation params and `squawk` that the method, its class and
|
|
33
|
+
`Squishling.config` add up to.
|
|
34
|
+
- **The harness:** that it is still registered (not removed or replaced since the class declared it), and that a
|
|
35
|
+
`type: :judgment` judge has an adapter that supports judgments.
|
|
36
|
+
- **Keywords:** that each `include_when:` and the `squish_when` block can be called with the method's inputs as keywords
|
|
37
|
+
(`instance_exec(**inputs, &proc)`): no required keyword that isn't an input, and no input a lambda (or a proc that names
|
|
38
|
+
keywords) can't take. Add `**` to accept any. A method that takes `**` or has no implementation can receive any
|
|
39
|
+
keyword, so nothing is reported for it. `squish_fallback` and `squish_validate` are called with the inputs too, but
|
|
40
|
+
aren't checked.
|
|
41
|
+
- **The adapter** (once): that `config.adapter`, or RubyLLM's, can be built.
|
|
42
|
+
|
|
43
|
+
A problem in an inherited method is reported once for each subclass, since each is checked with its own settings.
|
|
44
|
+
|
|
45
|
+
Nothing is sent to a provider, and your own code (the Ruby implementation, `squish_when`, the procs) is not run.
|
|
46
|
+
|
|
47
|
+
## Only loaded classes are checked
|
|
48
|
+
|
|
49
|
+
The list holds the classes that have been loaded. In Rails, load everything first:
|
|
50
|
+
|
|
51
|
+
```ruby
|
|
52
|
+
Rails.application.eager_load!
|
|
53
|
+
Squishling.verify!
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
The list holds its classes weakly, so it doesn't keep classes alive across code reloads, and a named class whose
|
|
57
|
+
constant now holds a different object (replaced by a reload but not yet collected) is left out. Anonymous classes are
|
|
58
|
+
included. `Squishling.squished_classes` returns the list.
|
|
59
|
+
|
|
60
|
+
## In CI
|
|
61
|
+
|
|
62
|
+
**Rake.** In Rails the `squishling:verify` task is added for you and eager loads the app first. Elsewhere, load it in your
|
|
63
|
+
`Rakefile`:
|
|
64
|
+
|
|
65
|
+
```ruby
|
|
66
|
+
load "squishling/tasks/squishling.rake"
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
`bundle exec rake squishling:verify` prints the report and exits non-zero when there are problems. Load your classes
|
|
70
|
+
before the task runs (an `:environment` task, if you have one, is invoked first). The Railtie adds the task in Rails
|
|
71
|
+
when Squishling is required after Rails (as Bundler does), so a Rails app needn't load the file itself.
|
|
72
|
+
|
|
73
|
+
**RSpec.** `require "squishling/rspec"` adds a matcher whose failure message is the report:
|
|
74
|
+
|
|
75
|
+
```ruby
|
|
76
|
+
require "squishling/rspec"
|
|
77
|
+
|
|
78
|
+
RSpec.describe "squished classes" do
|
|
79
|
+
before(:all) { Rails.application.eager_load! }
|
|
80
|
+
|
|
81
|
+
it { expect(Squishling).to be_squishling_valid }
|
|
82
|
+
it { expect(InvoiceParser).to be_squishling_valid } # or just one class, or an Array of them
|
|
83
|
+
end
|
|
84
|
+
```
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Squishling
|
|
4
|
+
module Adapter
|
|
5
|
+
# Reads the machine-readable fields of a provider's error body: the status (OpenAI-style `type`, Gemini's
|
|
6
|
+
# `status`), the code, and the name of the field it objected to. Only short plain tokens come out; the message
|
|
7
|
+
# and anything else a provider might echo from the request never do.
|
|
8
|
+
module ErrorBody
|
|
9
|
+
# A status or code is an enumeration-like word (no hyphens, so an API key like sk-... never passes); a param is
|
|
10
|
+
# a field path (messages[0].content), which is only compared with the developer's own param names and never
|
|
11
|
+
# repeated.
|
|
12
|
+
WORD = /\A[A-Za-z0-9][\w.]{0,39}\z/
|
|
13
|
+
PATH = /\A[\w.\-\[\]]{1,64}\z/
|
|
14
|
+
private_constant :WORD, :PATH
|
|
15
|
+
|
|
16
|
+
module_function
|
|
17
|
+
|
|
18
|
+
# body: a JSON String, a Hash with String keys, or an Array wrapping the Hash (Gemini does that). http_status
|
|
19
|
+
# drops a code that only repeats it. Returns { status:, code:, param: }, each a String or nil.
|
|
20
|
+
def read(body, http_status: nil)
|
|
21
|
+
detail = error(body)
|
|
22
|
+
return { status: nil, code: nil, param: nil } unless detail
|
|
23
|
+
|
|
24
|
+
code = token(detail["code"], WORD)
|
|
25
|
+
{ status: token(detail["status"], WORD) || token(detail["type"], WORD),
|
|
26
|
+
code: (code unless code == http_status.to_s),
|
|
27
|
+
param: token(detail["param"], PATH) }
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# The body's `error` Hash, or nil when it has none (or isn't JSON). For adapters that need more of it than
|
|
31
|
+
# #read returns, e.g. to recognize a wording; they must not repeat it.
|
|
32
|
+
def error(body)
|
|
33
|
+
data = body.is_a?(String) ? JSON.parse(body.dup.force_encoding(Encoding::UTF_8)) : body
|
|
34
|
+
data = data.first if data.is_a?(Array)
|
|
35
|
+
detail = data["error"] if data.is_a?(Hash)
|
|
36
|
+
detail if detail.is_a?(Hash)
|
|
37
|
+
rescue JSON::ParserError
|
|
38
|
+
nil
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def token(value, pattern)
|
|
42
|
+
value = value.to_s if value.is_a?(Integer)
|
|
43
|
+
value if value.is_a?(String) && value.match?(pattern)
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Squishling
|
|
4
|
+
# How Squishling reaches a model. An adapter is the only code that talks to a provider: Squishling builds a
|
|
5
|
+
# Request (the system prompt, the conversation so far, the strict output schema, the resolved params), the adapter
|
|
6
|
+
# makes one completion and returns a Response. Squishling owns the conversation, the validation, and the retries.
|
|
7
|
+
# Two adapters ship (Adapters::RubyLLM, Adapters::OpenAICompatible); write your own by subclassing Adapter::Base
|
|
8
|
+
# and registering it (Squishling.register_adapter) or assigning an instance to config.adapter. See docs/adapters.md.
|
|
9
|
+
module Adapter
|
|
10
|
+
# One completion to make. messages holds the conversation so far as { role: :user | :assistant, content: String }
|
|
11
|
+
# Hashes, ending with the user message to answer; instructions is the system prompt. schema is the strict
|
|
12
|
+
# structured-output payload ({ "name", "description", "schema", "strict" => true }). params is the resolved
|
|
13
|
+
# generation params: the portable :temperature, :max_output_tokens, and :thinking, plus any provider-specific keys
|
|
14
|
+
# (see Params). label ("Class#method") prefixes the adapter's error messages. model and provider are the step's
|
|
15
|
+
# (nil when it names none).
|
|
16
|
+
Request = Data.define(:label, :model, :provider, :instructions, :messages, :schema, :params)
|
|
17
|
+
|
|
18
|
+
# Stands in for an empty reply when the conversation is replayed: some servers reject an empty assistant message.
|
|
19
|
+
EMPTY_REPLY = "(no output)"
|
|
20
|
+
|
|
21
|
+
# What a completion returns. content is the model's output: the raw String, or a Hash the adapter already parsed.
|
|
22
|
+
# model is the model id the provider reports. tokens is a Hash of counts (:input_tokens, :output_tokens, and
|
|
23
|
+
# whatever else the provider reports, such as :cache_read_tokens and :thinking_tokens), empty when unknown.
|
|
24
|
+
# finish_reason is why the provider stopped, as a Symbol: :stop, :max_tokens (the output cap cut the reply off),
|
|
25
|
+
# :content_filter, :tool_calls, or the provider's own spelling; nil when it reported none.
|
|
26
|
+
Response = Data.define(:content, :model, :tokens, :finish_reason) do
|
|
27
|
+
# model, tokens, and finish_reason may be left out; nil tokens count as none, and a String or blank
|
|
28
|
+
# finish_reason is made a Symbol or nil. Raises ConfigurationError for tokens that aren't a Hash.
|
|
29
|
+
def initialize(content:, model: nil, tokens: {}, finish_reason: nil)
|
|
30
|
+
tokens ||= {}
|
|
31
|
+
unless tokens.is_a?(Hash)
|
|
32
|
+
raise ConfigurationError, "Adapter::Response tokens must be a Hash, got #{tokens.class}"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
finish_reason = finish_reason.to_s.strip.empty? ? nil : finish_reason.to_sym
|
|
36
|
+
super
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Whether the provider cut the reply off at the output token cap.
|
|
40
|
+
def truncated?
|
|
41
|
+
finish_reason == :max_tokens
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# The token counts, or nil when the adapter reported none (what squawk's usage: carries).
|
|
45
|
+
def usage
|
|
46
|
+
tokens unless tokens.empty?
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# The output as the text the model will see again in a retry: a Hash an adapter already parsed goes back as
|
|
50
|
+
# JSON, and nothing (or a value JSON can't represent, which is rejected as invalid output anyway) as
|
|
51
|
+
# EMPTY_REPLY.
|
|
52
|
+
def text
|
|
53
|
+
text = content.is_a?(String) ? utf8(content) : generated
|
|
54
|
+
text.strip.empty? ? EMPTY_REPLY : text
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
def generated
|
|
60
|
+
content.nil? ? "" : JSON.generate(content)
|
|
61
|
+
rescue JSON::JSONError
|
|
62
|
+
""
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Text that goes back out in a request must be valid UTF-8, or serializing the history fails. A reply that
|
|
66
|
+
# wasn't is already reported as invalid output; only its undecodable bytes are replaced here.
|
|
67
|
+
def utf8(text)
|
|
68
|
+
(text.encoding == Encoding::BINARY ? text.dup.force_encoding(Encoding::UTF_8) : text).scrub("?")
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# The adapter contract. Subclass it, implement #complete, and raise Squishling's errors:
|
|
73
|
+
#
|
|
74
|
+
# class MyAdapter < Squishling::Adapter::Base
|
|
75
|
+
# def complete(request)
|
|
76
|
+
# reply = MyClient.chat(request.model, request.instructions, request.messages)
|
|
77
|
+
# Squishling::Adapter::Response.new(content: reply.text, model: reply.model, tokens: reply.usage)
|
|
78
|
+
# rescue MyClient::Unauthorized
|
|
79
|
+
# raise configuration_error(request, "MyClient rejected the credentials")
|
|
80
|
+
# rescue MyClient::Error => e
|
|
81
|
+
# raise llm_error(request, "MyClient failed (#{e.class})")
|
|
82
|
+
# end
|
|
83
|
+
# end
|
|
84
|
+
#
|
|
85
|
+
# ConfigurationError is for setup mistakes (bad credentials, an invalid request, an unknown model): it is never
|
|
86
|
+
# retried or escalated. LLMError is for failures worth another attempt (rate limits, 5xx, timeouts, connection
|
|
87
|
+
# errors): the next attempt, or the last one's caller, sees it. Keep the provider's own message out of both: it can
|
|
88
|
+
# echo request content or part of a key. Any other exception propagates unwrapped. Adapters are called from worker
|
|
89
|
+
# threads when a harness takes samples, so they must not share unsynchronized state.
|
|
90
|
+
class Base
|
|
91
|
+
# One completion. Returns an Adapter::Response.
|
|
92
|
+
def complete(_request)
|
|
93
|
+
raise NotImplementedError, "#{self.class} must implement #complete(request)"
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# Whether #judge is supported (judges with type: :judgment).
|
|
97
|
+
def judgments?
|
|
98
|
+
false
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# A System One judgment: input (a Hash) and questions (a Hash of name => question), with the step's model and
|
|
102
|
+
# params. Returns an object answering [name] with an answer that responds to choice and probabilities. label
|
|
103
|
+
# ("Class#method") prefixes error messages. Called as judge(input, questions:, step:, label:).
|
|
104
|
+
def judge(_input, **_options)
|
|
105
|
+
raise ConfigurationError, "#{self.class} doesn't support judgments (type: :judgment judges)"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Called on the caller's thread before a harness's sample requests fan out to worker threads. Returns whatever
|
|
109
|
+
# #with_context needs to re-establish the caller's request context (tracing, say) on a worker.
|
|
110
|
+
def capture_context
|
|
111
|
+
nil
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Runs the block on a worker thread inside the context #capture_context returned.
|
|
115
|
+
def with_context(_context)
|
|
116
|
+
yield
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
private
|
|
120
|
+
|
|
121
|
+
def configuration_error(request, message)
|
|
122
|
+
ConfigurationError.new("#{request.label}: #{message}")
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def llm_error(request, message)
|
|
126
|
+
LLMError.new("#{request.label}: #{message}")
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# A RequestRejectedError for a request the provider rejected (a 400). via names how it failed (the error class,
|
|
130
|
+
# the HTTP status); body is the provider's error body (a JSON String, or a Hash with String keys), of which only
|
|
131
|
+
# the status and code tokens are repeated, never its message. http_status, when given, drops a code that only
|
|
132
|
+
# repeats it. The params are pointed at only when the provider named one of them.
|
|
133
|
+
def rejected_request_error(request, via:, body:, http_status: nil)
|
|
134
|
+
found = ErrorBody.read(body, http_status:)
|
|
135
|
+
tokens = found.values_at(:status, :code).compact
|
|
136
|
+
RequestRejectedError.new(
|
|
137
|
+
"#{request.label}: the provider rejected the request (#{[via, *tokens].join(', ')}). " \
|
|
138
|
+
"#{rejection_hint(request.params, found[:param])}",
|
|
139
|
+
provider_status: found[:status], provider_code: found[:code]
|
|
140
|
+
)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def rejection_hint(params, field)
|
|
144
|
+
named = params.keys.find { |key| names?(field, key.to_s) }
|
|
145
|
+
return "The #{suspects(params)} may be invalid." unless named
|
|
146
|
+
|
|
147
|
+
"The provider named the #{named.inspect} param; check its value."
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def suspects(params)
|
|
151
|
+
params.empty? ? "output schema or input" : "params, output schema, or input"
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# Whether the provider's field is the param or inside it (thinking.budget_tokens, stop[0]).
|
|
155
|
+
def names?(field, name)
|
|
156
|
+
!field.nil? && (field == name || field.start_with?("#{name}.", "#{name}["))
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# The adapters by name: the built-ins, registered as they load, and your own (Squishling.register_adapter).
|
|
161
|
+
module Registry
|
|
162
|
+
STORE = NamedRegistry.new("adapter")
|
|
163
|
+
private_constant :STORE
|
|
164
|
+
|
|
165
|
+
class << self
|
|
166
|
+
def register_built_in(name, klass)
|
|
167
|
+
STORE.register_built_in(name, klass)
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# Registers an adapter class under a name. It is built with the options given next to its type: (or none).
|
|
171
|
+
def register(name, klass)
|
|
172
|
+
name = STORE.checked_name(name)
|
|
173
|
+
unless implements_complete?(klass)
|
|
174
|
+
raise ConfigurationError, "register_adapter needs a class that implements #complete, got #{klass.inspect}"
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
STORE.add(name, klass)
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def fetch(name)
|
|
181
|
+
STORE.fetch(name)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
def names
|
|
185
|
+
STORE.names
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# Forgets the registered adapters; the built-ins stay.
|
|
189
|
+
def reset!
|
|
190
|
+
STORE.reset!
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
private
|
|
194
|
+
|
|
195
|
+
# A class that defines #complete itself, not one that only inherits Base's placeholder.
|
|
196
|
+
def implements_complete?(klass)
|
|
197
|
+
klass.is_a?(Class) && klass.method_defined?(:complete) && klass.instance_method(:complete).owner != Base
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
KEYWORD_KINDS = %i[key keyreq].freeze
|
|
203
|
+
private_constant :KEYWORD_KINDS
|
|
204
|
+
|
|
205
|
+
module_function
|
|
206
|
+
|
|
207
|
+
# The adapter a config.adapter value names: an instance (anything that responds to complete), a registered name,
|
|
208
|
+
# or a Hash with type: and that adapter's options. Built now, so a mistake (a missing gem, a bad option) fails at
|
|
209
|
+
# setup rather than on the first request.
|
|
210
|
+
def build(value, where)
|
|
211
|
+
case value
|
|
212
|
+
when Symbol, String then instantiate(value, {}, where)
|
|
213
|
+
when Hash
|
|
214
|
+
options = value.transform_keys(&:to_sym)
|
|
215
|
+
raise ConfigurationError, "#{where} adapter: needs a type: (#{type_list})" unless options.key?(:type)
|
|
216
|
+
|
|
217
|
+
instantiate(options.delete(:type), options, where)
|
|
218
|
+
else
|
|
219
|
+
return value if value.respond_to?(:complete)
|
|
220
|
+
|
|
221
|
+
raise ConfigurationError, "#{where} adapter must be an adapter (responding to complete), one of " \
|
|
222
|
+
"#{type_list}, or a Hash with type:, got #{value.inspect}"
|
|
223
|
+
end
|
|
224
|
+
end
|
|
225
|
+
|
|
226
|
+
# The adapter used when none is configured: RubyLLM's, if the ruby_llm gem is available.
|
|
227
|
+
def detect
|
|
228
|
+
Adapters::RubyLLM.new
|
|
229
|
+
rescue Adapters::RubyLLM::MissingGemError
|
|
230
|
+
raise ConfigurationError, "No adapter is configured and the ruby_llm gem isn't available. " \
|
|
231
|
+
"Add `gem \"ruby_llm\"` to your Gemfile, or set config.adapter " \
|
|
232
|
+
"(#{type_list}, or your own adapter)"
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
def type_list
|
|
236
|
+
Registry.names.map(&:inspect).join(", ")
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def instantiate(type, options, where)
|
|
240
|
+
name = type.to_s.to_sym
|
|
241
|
+
klass = Registry.fetch(name)
|
|
242
|
+
unless klass
|
|
243
|
+
raise ConfigurationError, "#{where} adapter: unknown adapter #{type.inspect} (available: #{type_list})"
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
check_options!(klass, options, name, where)
|
|
247
|
+
options.empty? ? klass.new : klass.new(**options)
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
# The keywords the adapter's initialize takes, so a typo is a ConfigurationError rather than an ArgumentError.
|
|
251
|
+
def check_options!(klass, options, name, where)
|
|
252
|
+
known, required = keywords(klass)
|
|
253
|
+
return unless known
|
|
254
|
+
|
|
255
|
+
unknown = options.keys - known
|
|
256
|
+
missing = required - options.keys
|
|
257
|
+
return if unknown.empty? && missing.empty?
|
|
258
|
+
|
|
259
|
+
problem = unknown.empty? ? "needs #{missing.join(', ')}" : "doesn't take #{unknown.join(', ')}"
|
|
260
|
+
hint = known.empty? ? "" : " (options: #{known.join(', ')})"
|
|
261
|
+
raise ConfigurationError, "#{where} adapter #{name.inspect} #{problem}#{hint}"
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
# [all keywords, required keywords] of klass#initialize, or nil when it takes any (**options).
|
|
265
|
+
def keywords(klass)
|
|
266
|
+
parameters = klass.instance_method(:initialize).parameters
|
|
267
|
+
return if parameters.any? { |kind, _| kind == :keyrest }
|
|
268
|
+
|
|
269
|
+
[parameters.filter_map { |kind, key| key if KEYWORD_KINDS.include?(kind) },
|
|
270
|
+
parameters.filter_map { |kind, key| key if kind == :keyreq }]
|
|
271
|
+
end
|
|
272
|
+
private_class_method :check_options!, :keywords
|
|
273
|
+
private_class_method :instantiate
|
|
274
|
+
end
|
|
275
|
+
end
|