jevalyn 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +45 -0
- data/LICENSE.txt +21 -0
- data/README.md +533 -0
- data/lib/generators/jevalyn/decision/decision_generator.rb +60 -0
- data/lib/generators/jevalyn/decision/templates/decision.rb.tt +53 -0
- data/lib/generators/jevalyn/decision/templates/decision_spec.rb.tt +35 -0
- data/lib/generators/jevalyn/guardrail/guardrail_generator.rb +36 -0
- data/lib/generators/jevalyn/guardrail/templates/guardrail.rb.tt +23 -0
- data/lib/generators/jevalyn/guardrail/templates/guardrail_spec.rb.tt +26 -0
- data/lib/generators/jevalyn/install/install_generator.rb +37 -0
- data/lib/generators/jevalyn/install/templates/jevalyn.rb.tt +29 -0
- data/lib/jevalyn/answer.rb +176 -0
- data/lib/jevalyn/client.rb +167 -0
- data/lib/jevalyn/configuration.rb +85 -0
- data/lib/jevalyn/decision.rb +238 -0
- data/lib/jevalyn/errors.rb +123 -0
- data/lib/jevalyn/evaluation_job.rb +79 -0
- data/lib/jevalyn/guardrail.rb +130 -0
- data/lib/jevalyn/question.rb +184 -0
- data/lib/jevalyn/railtie.rb +37 -0
- data/lib/jevalyn/result.rb +164 -0
- data/lib/jevalyn/router.rb +175 -0
- data/lib/jevalyn/state.rb +80 -0
- data/lib/jevalyn/state_adapters/active_record_adapter.rb +26 -0
- data/lib/jevalyn/tasks/jevalyn.rake +35 -0
- data/lib/jevalyn/testing/cassette.rb +94 -0
- data/lib/jevalyn/testing/minitest.rb +55 -0
- data/lib/jevalyn/testing/rspec.rb +118 -0
- data/lib/jevalyn/testing.rb +276 -0
- data/lib/jevalyn/version.rb +5 -0
- data/lib/jevalyn.rb +88 -0
- metadata +123 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 8296f5db8e42556a58e6b5ef934d3c64aea9fb3397ff33e86636dc077655baab
|
|
4
|
+
data.tar.gz: dde694da23fd37c22a096cc3f5a5bdb48dba36c262277158a7234aef356772ad
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 3a5e965688e40b0abdd050950a8f0b1f45c985229db3cd2164c98aab6f7d5ff21afe99becae9a977bb2c9bf52ecc2ed7629dc3ec089a20fc42f889e7b81e6a57
|
|
7
|
+
data.tar.gz: 66552bcf4efc05786b06b83e926e117ae635e480c2e55b0e8127a20c0917026dfbebd3a5ed849eab6ab6fce85ec6189a674e4544c172ca0ae69179122402bfc0
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project are documented here. This project follows
|
|
4
|
+
[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
5
|
+
|
|
6
|
+
## [Unreleased]
|
|
7
|
+
|
|
8
|
+
## [0.1.0]
|
|
9
|
+
|
|
10
|
+
First release. Built against the `jev-1.13` HTTP API.
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- `Jevalyn::Client` — a one-to-one wrapper over `POST /v1/systemone`, with typed errors,
|
|
15
|
+
automatic backoff on 429/529/5xx, and `retry-after` support.
|
|
16
|
+
- `Jevalyn::Decision` — the question DSL, validated at class-definition time, with a
|
|
17
|
+
generated reader per question on the result.
|
|
18
|
+
- Per-question confidence floors: `confidence_threshold:` on a question, with the
|
|
19
|
+
class-level `confidence_threshold` as the default for questions that omit it.
|
|
20
|
+
`thresholds:` overrides individual questions at the call site, `confidence_threshold:`
|
|
21
|
+
overrides all of them, and `Result#certainty_margins` reports how far each answer
|
|
22
|
+
sat from its own floor.
|
|
23
|
+
- `Jevalyn::Answer` — `Noul`, `Choice` and `Score` wrappers over the three answer shapes.
|
|
24
|
+
- `Jevalyn::Guardrail` — a single-noul gate answering `#allow?` / `#deny?`, denying rather
|
|
25
|
+
than failing open when the API is unreachable.
|
|
26
|
+
- `Jevalyn::Router` — dispatch on a choice answer, with a confidence floor and
|
|
27
|
+
build-time checking that every option has a route.
|
|
28
|
+
- `Jevalyn::State` — the `#jevalyn_state` convention, with an ActiveRecord adapter.
|
|
29
|
+
- `Jevalyn::Testing` — local stubs, RSpec and Minitest integration, and a cassette
|
|
30
|
+
recorder keyed on request content. `confidence:` takes a Hash so one stub can put
|
|
31
|
+
one answer under its floor while another clears its own.
|
|
32
|
+
- `evaluate_later` for ActiveJob.
|
|
33
|
+
- Generators: `jevalyn:install`, `jevalyn:decision`, `jevalyn:guardrail`.
|
|
34
|
+
- Rake tasks: `jevalyn:ping`, `jevalyn:decisions`.
|
|
35
|
+
|
|
36
|
+
### Notes
|
|
37
|
+
|
|
38
|
+
Two details of the API are easy to get wrong, and Jevalyn models them explicitly:
|
|
39
|
+
|
|
40
|
+
- **Noul answers carry no `confidence`.** TypeSafe returns `confidence` on Choice and
|
|
41
|
+
Score answers only. `Answer::Noul#confidence` is `nil`; `#certainty` derives a
|
|
42
|
+
comparable number from the probability's distance off `0.5` so thresholds work
|
|
43
|
+
uniformly.
|
|
44
|
+
- **Score answers are a weighted Float, not a level label.** `2.4` lands between levels
|
|
45
|
+
on purpose. `#label` gives the nearest level's description.
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Raymond Hughes
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,533 @@
|
|
|
1
|
+
<div align="center">
|
|
2
|
+
|
|
3
|
+
<picture>
|
|
4
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/Ray-Hughes/jevalyn/main/docs/assets/logo-dark.png">
|
|
5
|
+
<img src="https://raw.githubusercontent.com/Ray-Hughes/jevalyn/main/docs/assets/logo.png" alt="Jevalyn" width="340">
|
|
6
|
+
</picture>
|
|
7
|
+
|
|
8
|
+
**Fast, cheap, structured decisions baked into your Rails app's control flow.**
|
|
9
|
+
|
|
10
|
+
[](https://rubygems.org/gems/jevalyn)
|
|
11
|
+
[](https://rubygems.org/gems/jevalyn)
|
|
12
|
+
[](https://github.com/Ray-Hughes/jevalyn/actions/workflows/ci.yml)
|
|
13
|
+
[](https://www.ruby-lang.org)
|
|
14
|
+
[](https://rubyonrails.org)
|
|
15
|
+
[](https://docs.typesafe.ai)
|
|
16
|
+
[](LICENSE.txt)
|
|
17
|
+
|
|
18
|
+
[Install](#install) · [Decisions](#decisions) · [Guardrails](#guardrails) · [Router](#router) · [Testing](#testing)
|
|
19
|
+
|
|
20
|
+
</div>
|
|
21
|
+
|
|
22
|
+
---
|
|
23
|
+
|
|
24
|
+
```ruby
|
|
25
|
+
class SupportTriage < Jevalyn::Decision
|
|
26
|
+
question :department, type: :choice,
|
|
27
|
+
instructions: "Which team should handle this?",
|
|
28
|
+
criteria: {
|
|
29
|
+
billing: "Payments, invoicing, refunds",
|
|
30
|
+
technical: "Bugs, outages, integrations",
|
|
31
|
+
sales: "Pricing, upgrades, new accounts"
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
confidence_threshold 0.75
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
result = SupportTriage.evaluate(ticket.body)
|
|
38
|
+
|
|
39
|
+
result.department # => :technical
|
|
40
|
+
result.certain? # => true
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
**Jevalyn is the decision layer for your Rails app**, built on
|
|
44
|
+
[Jev](https://docs.typesafe.ai). That is a routing decision made by a model, inside a
|
|
45
|
+
Rails request, in about as long as a database query. Not a prompt, not a parsed JSON
|
|
46
|
+
blob, not a retry loop around something that might return prose this time.
|
|
47
|
+
|
|
48
|
+
## What this is, and what it is not
|
|
49
|
+
|
|
50
|
+
Jevalyn wraps [TypeSafe's Jev](https://docs.typesafe.ai), a **System One** model. You
|
|
51
|
+
give it a `state` and a set of typed questions; it gives back typed, calibrated
|
|
52
|
+
answers. There are exactly three kinds of question:
|
|
53
|
+
|
|
54
|
+
| Type | Ask it | Get back |
|
|
55
|
+
| --- | --- | --- |
|
|
56
|
+
| `noul` | a yes/no question | the probability the answer is yes, `0.0`–`1.0` |
|
|
57
|
+
| `choice` | pick one of up to 255 options you define | the winner, the full distribution, a confidence |
|
|
58
|
+
| `score` | rate against 2–10 ordered levels you define | a weighted score, the distribution, a confidence |
|
|
59
|
+
|
|
60
|
+
**Jev does not generate text.** No summaries, no drafts, no open-ended reasoning, no
|
|
61
|
+
tool calls. If you need prose, you need an LLM, and Jevalyn will happily route to one
|
|
62
|
+
(see [Router](#router)) — but it will not pretend to be one.
|
|
63
|
+
|
|
64
|
+
What it is good at is the decision *around* the work: which queue does this belong in,
|
|
65
|
+
is this safe to auto-approve, how severe is this, does this comment need a human, which
|
|
66
|
+
of these 200 documents actually answers the question. Those are cheap, fast, and
|
|
67
|
+
type-safe here, and expensive, slow, and stringly-typed anywhere else.
|
|
68
|
+
|
|
69
|
+
You can build anything on top of it — trade screening, recipe filtering, content
|
|
70
|
+
moderation. "Use Jevalyn for recipes" means *classifying and scoring* recipes, not
|
|
71
|
+
writing them.
|
|
72
|
+
|
|
73
|
+
## Install
|
|
74
|
+
|
|
75
|
+
```ruby
|
|
76
|
+
# Gemfile
|
|
77
|
+
gem "jevalyn"
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
```console
|
|
81
|
+
$ bundle install
|
|
82
|
+
$ bin/rails g jevalyn:install
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
That writes `config/initializers/jevalyn.rb` and creates `app/decisions/`. Then put
|
|
86
|
+
your key in the environment:
|
|
87
|
+
|
|
88
|
+
```console
|
|
89
|
+
$ export TYPESAFE_API_KEY=ts_...
|
|
90
|
+
$ bin/rails jevalyn:ping
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Get a key at [typesafe.ai](https://typesafe.ai). Jev bills on **input tokens only** —
|
|
94
|
+
output tokens are free — at roughly $0.042 per million input tokens as of `jev-1.13`.
|
|
95
|
+
A support ticket costs a fraction of a cent to triage. Check
|
|
96
|
+
[the models page](https://docs.typesafe.ai/models) for current pricing and rate limits;
|
|
97
|
+
they are still moving.
|
|
98
|
+
|
|
99
|
+
## Decisions
|
|
100
|
+
|
|
101
|
+
A `Jevalyn::Decision` is a named set of questions your app asks about a piece of state.
|
|
102
|
+
|
|
103
|
+
```console
|
|
104
|
+
$ bin/rails g jevalyn:decision SupportTriage urgent:noul department:choice severity:score
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
```ruby
|
|
108
|
+
class SupportTriage < Jevalyn::Decision
|
|
109
|
+
question :urgent, type: :noul,
|
|
110
|
+
instructions: "Does this convey urgency?",
|
|
111
|
+
criteria: { true: "Explicitly time-sensitive", false: "No urgency expressed" }
|
|
112
|
+
|
|
113
|
+
question :department, type: :choice,
|
|
114
|
+
instructions: "Which team should handle this?",
|
|
115
|
+
criteria: {
|
|
116
|
+
billing: "Payments, invoicing, refunds",
|
|
117
|
+
technical: "Bugs, outages, integrations",
|
|
118
|
+
sales: "Pricing, upgrades, new accounts"
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
question :severity, type: :score,
|
|
122
|
+
instructions: "How severe is this issue?",
|
|
123
|
+
criteria: ["trivial", "minor", "major", "critical"]
|
|
124
|
+
|
|
125
|
+
confidence_threshold 0.75
|
|
126
|
+
end
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
Every question is validated when the class body runs, so a rubric with eleven score
|
|
130
|
+
levels or a `:choice` with no criteria fails on boot — not as a 422 on a Friday
|
|
131
|
+
afternoon.
|
|
132
|
+
|
|
133
|
+
All three questions go out in **one request**. Jev reads the state once and evaluates
|
|
134
|
+
every question against it in parallel, which is both cheaper and faster than asking
|
|
135
|
+
three times. Add speculative questions freely; they cost a few tokens each.
|
|
136
|
+
|
|
137
|
+
### Reading the result
|
|
138
|
+
|
|
139
|
+
```ruby
|
|
140
|
+
result = SupportTriage.evaluate(ticket.body)
|
|
141
|
+
|
|
142
|
+
result.urgent # => 0.92 the raw probability
|
|
143
|
+
result.urgent? # => true at the default 0.5 cutoff
|
|
144
|
+
result.urgent?(0.95) # => false your cutoff, your call
|
|
145
|
+
|
|
146
|
+
result.department # => :technical
|
|
147
|
+
result.department_confidence # => 0.82
|
|
148
|
+
result.department_probabilities # => { "billing" => 0.08, "technical" => 0.85, ... }
|
|
149
|
+
result.department_answer.runner_up # => :billing
|
|
150
|
+
|
|
151
|
+
result.severity # => 2.4 weighted, lands between levels
|
|
152
|
+
result.severity_label # => "major" nearest level
|
|
153
|
+
result.severity_level # => 2
|
|
154
|
+
|
|
155
|
+
result.certain? # => true every answer cleared 0.75
|
|
156
|
+
result.uncertain_questions # => [] or the names that did not
|
|
157
|
+
result.model # => "jev-1.13.0"
|
|
158
|
+
result.input_tokens # => 312
|
|
159
|
+
result.values # => { urgent: 0.92, department: :technical, severity: 2.4 }
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
Two things here differ from what you might expect, and both come straight from the API:
|
|
163
|
+
|
|
164
|
+
- **A score is a Float, not a label.** `2.4` means past *major* and heading for
|
|
165
|
+
*critical*. Use `severity_label` when you want the nearest level's name, and
|
|
166
|
+
`severity` itself when you want to do arithmetic.
|
|
167
|
+
- **A noul has no confidence.** Jev returns `confidence` on Choice and Score answers
|
|
168
|
+
only — for a noul, the probability *is* the answer, and `0.5` is the model telling
|
|
169
|
+
you it does not know. Jevalyn fills the gap with `certainty`, derived from how far
|
|
170
|
+
the value sits from a coin flip, so `certain?` works uniformly across all three
|
|
171
|
+
types. That number is Jevalyn's arithmetic, not TypeSafe's; `confidence` stays `nil`
|
|
172
|
+
so you always know which is which.
|
|
173
|
+
|
|
174
|
+
### Confidence
|
|
175
|
+
|
|
176
|
+
Confidence is the second axis. The answer tells you *what*; confidence tells you
|
|
177
|
+
*whether to act*.
|
|
178
|
+
|
|
179
|
+
```ruby
|
|
180
|
+
result = RefundDecision.evaluate(request)
|
|
181
|
+
|
|
182
|
+
if result.uncertain?
|
|
183
|
+
HumanReview.enqueue(request) # the model said "I'm not sure"
|
|
184
|
+
elsif result.approve?
|
|
185
|
+
Refund.issue(request)
|
|
186
|
+
end
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
A threshold is not one number for your whole app — and usually not one number for a
|
|
190
|
+
whole decision either. Misrouting a ticket is recoverable; sending it to a team that
|
|
191
|
+
cannot help is worse; auto-approving a refund is worse again. So a floor belongs to
|
|
192
|
+
the **question**, and `confidence_threshold` on the class is just the default for
|
|
193
|
+
questions that do not name their own:
|
|
194
|
+
|
|
195
|
+
```ruby
|
|
196
|
+
class SupportTriage < Jevalyn::Decision
|
|
197
|
+
question :department, type: :choice,
|
|
198
|
+
instructions: "Which team should handle this?",
|
|
199
|
+
criteria: { ... },
|
|
200
|
+
confidence_threshold: 0.8 # routing to the wrong team wastes a day
|
|
201
|
+
|
|
202
|
+
question :severity, type: :score,
|
|
203
|
+
instructions: "How severe is this issue?",
|
|
204
|
+
criteria: ["trivial", "minor", "major", "critical"],
|
|
205
|
+
confidence_threshold: 0.6 # a roughly-right severity is still useful
|
|
206
|
+
|
|
207
|
+
confidence_threshold 0.75 # the default for anything above without one
|
|
208
|
+
end
|
|
209
|
+
```
|
|
210
|
+
|
|
211
|
+
Each answer is then judged against its own floor:
|
|
212
|
+
|
|
213
|
+
```ruby
|
|
214
|
+
result = SupportTriage.evaluate(ticket.body)
|
|
215
|
+
|
|
216
|
+
result.certain? # => false — every answer against its own floor
|
|
217
|
+
result.uncertain_questions # => [:department] — severity was fine at 0.65
|
|
218
|
+
result.department_certain? # => false
|
|
219
|
+
result.severity_certain? # => true
|
|
220
|
+
|
|
221
|
+
result.thresholds # => { urgent: 0.75, department: 0.8, severity: 0.6 }
|
|
222
|
+
result.department_threshold # => 0.8
|
|
223
|
+
result.certainty_margins # => { urgent: 0.15, department: -0.1, severity: 0.05 }
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
`certainty_margins` is how far each answer sits above its floor — negative means it
|
|
227
|
+
missed. Log it for a week and you will know which floors you actually set correctly.
|
|
228
|
+
|
|
229
|
+
Override at the call site when the stakes change for one call. `confidence_threshold:`
|
|
230
|
+
applies one number to everything; `thresholds:` names questions individually and wins
|
|
231
|
+
where both are given:
|
|
232
|
+
|
|
233
|
+
```ruby
|
|
234
|
+
SupportTriage.evaluate(body, confidence_threshold: 0.95) # stricter about all of it
|
|
235
|
+
SupportTriage.evaluate(body, thresholds: { department: 0.9 }) # stricter about one
|
|
236
|
+
```
|
|
237
|
+
|
|
238
|
+
So the floor for a question resolves, most specific first:
|
|
239
|
+
|
|
240
|
+
1. `thresholds:` at the call site
|
|
241
|
+
2. `confidence_threshold:` at the call site
|
|
242
|
+
3. `confidence_threshold:` on the question
|
|
243
|
+
4. `confidence_threshold` on the decision
|
|
244
|
+
5. `Jevalyn.config.default_confidence_threshold`
|
|
245
|
+
|
|
246
|
+
With none of them set, `certain?` is `true` — nothing was asked for.
|
|
247
|
+
|
|
248
|
+
A noul carries no confidence, so its floor is measured against `certainty`: how far
|
|
249
|
+
the probability sits from a coin flip. `0.6` against a floor of `0.75` is uncertain;
|
|
250
|
+
`0.92` clears it.
|
|
251
|
+
|
|
252
|
+
### State
|
|
253
|
+
|
|
254
|
+
Anything JSON-shaped works: a String, a Hash, an Array, an ActiveRecord model.
|
|
255
|
+
|
|
256
|
+
```ruby
|
|
257
|
+
SupportTriage.evaluate(ticket.body) # a string
|
|
258
|
+
SupportTriage.evaluate({ subject: s, body: b, plan: "pro" }) # a hash
|
|
259
|
+
SupportTriage.evaluate(conversation.messages) # an array
|
|
260
|
+
SupportTriage.evaluate(ticket) # an AR model
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
For a model, define `#jevalyn_state` and send only what the decision needs. The default
|
|
264
|
+
is `as_json`, which ships every column — wasteful in tokens and careless with data that
|
|
265
|
+
did not need to leave the building.
|
|
266
|
+
|
|
267
|
+
```ruby
|
|
268
|
+
class Ticket < ApplicationRecord
|
|
269
|
+
def jevalyn_state
|
|
270
|
+
{ subject:, body:, plan: account.plan, previous_tickets: account.tickets.count }
|
|
271
|
+
end
|
|
272
|
+
end
|
|
273
|
+
```
|
|
274
|
+
|
|
275
|
+
Jev's budget is 64k tokens per request, and 32k for the state plus the longest single
|
|
276
|
+
question. `SupportTriage.estimated_tokens(state)` gives a rough count before you send.
|
|
277
|
+
|
|
278
|
+
### Checking cost before you ship
|
|
279
|
+
|
|
280
|
+
```ruby
|
|
281
|
+
SupportTriage.payload_for(ticket.body) # exactly what would be sent, unsent
|
|
282
|
+
SupportTriage.estimated_tokens(ticket.body)
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
## Guardrails
|
|
286
|
+
|
|
287
|
+
A `Guardrail` is a Decision narrowed to one job: should this be allowed through?
|
|
288
|
+
|
|
289
|
+
```console
|
|
290
|
+
$ bin/rails g jevalyn:guardrail ToolCall
|
|
291
|
+
```
|
|
292
|
+
|
|
293
|
+
```ruby
|
|
294
|
+
class ToolCallGuardrail < Jevalyn::Guardrail
|
|
295
|
+
question :safe_to_execute, type: :noul,
|
|
296
|
+
instructions: "Is this tool call safe to run without human review?",
|
|
297
|
+
criteria: {
|
|
298
|
+
true: "Read-only, scoped to the current user's own data",
|
|
299
|
+
false: "Writes, deletes, spends money, or touches another account"
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
allow_above 0.95
|
|
303
|
+
on_error :deny
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
ToolCallGuardrail.check(tool_call).allow?
|
|
307
|
+
```
|
|
308
|
+
|
|
309
|
+
One noul question, and `allow_above` is the probability it must reach. The default is
|
|
310
|
+
`0.5` — a coin flip — which is almost certainly not what you want in front of anything
|
|
311
|
+
destructive.
|
|
312
|
+
|
|
313
|
+
If the API call itself fails, a guardrail **denies** rather than failing open:
|
|
314
|
+
|
|
315
|
+
```ruby
|
|
316
|
+
result = ToolCallGuardrail.check(payload)
|
|
317
|
+
|
|
318
|
+
result.deny? # => true
|
|
319
|
+
result.failed? # => true — denied because TypeSafe was unreachable
|
|
320
|
+
result.error # => #<Jevalyn::OverloadedError ...>
|
|
321
|
+
```
|
|
322
|
+
|
|
323
|
+
Set `on_error :raise` if you would rather handle the outage yourself.
|
|
324
|
+
|
|
325
|
+
## Router
|
|
326
|
+
|
|
327
|
+
A dispatch table with a confidence floor underneath it. This is where Jev hands off to
|
|
328
|
+
something slower when it is not sure.
|
|
329
|
+
|
|
330
|
+
```ruby
|
|
331
|
+
router = Jevalyn::Router.new(SupportTriage, on: :department) do |r|
|
|
332
|
+
r.route :billing, to: BillingInbox
|
|
333
|
+
r.route :technical, to: ->(ticket, result) { Oncall.page(ticket, result.severity) }
|
|
334
|
+
r.route :sales, to: SalesInbox
|
|
335
|
+
|
|
336
|
+
r.uncertain_below 0.75, to: HumanQueue
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
router.call(ticket)
|
|
340
|
+
```
|
|
341
|
+
|
|
342
|
+
Handlers are anything responding to `#call`, or another `Jevalyn::Decision`. They get
|
|
343
|
+
`(state, result)` if they take two arguments and `(state)` if they take one.
|
|
344
|
+
|
|
345
|
+
The router checks at build time that every option your `:choice` can return has a route
|
|
346
|
+
— so adding a fourth department and forgetting to route it is a boot error, not a
|
|
347
|
+
production exception on an unusual ticket.
|
|
348
|
+
|
|
349
|
+
Without a decision it is a plain dispatch table:
|
|
350
|
+
|
|
351
|
+
```ruby
|
|
352
|
+
router = Jevalyn::Router.new do |r|
|
|
353
|
+
r.route :lookup, to: OrderLookup
|
|
354
|
+
r.route :reason, to: llm_client
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
router.dispatch(:lookup, state: order)
|
|
358
|
+
```
|
|
359
|
+
|
|
360
|
+
Keep it thin. If a route needs branching logic, that logic belongs in the handler.
|
|
361
|
+
|
|
362
|
+
## Testing
|
|
363
|
+
|
|
364
|
+
TypeSafe has no sandbox key, so `mock_mode` is entirely local: with it on, the client
|
|
365
|
+
never opens a connection.
|
|
366
|
+
|
|
367
|
+
```ruby
|
|
368
|
+
# spec/spec_helper.rb
|
|
369
|
+
require "jevalyn/testing/rspec"
|
|
370
|
+
```
|
|
371
|
+
|
|
372
|
+
That turns mock mode on for the suite, resets stubs between examples, and adds helpers:
|
|
373
|
+
|
|
374
|
+
```ruby
|
|
375
|
+
it "routes a payment failure to the technical team" do
|
|
376
|
+
stub_jevalyn(SupportTriage, urgent: true, department: :technical, severity: "major")
|
|
377
|
+
|
|
378
|
+
expect(TriageJob.perform_now(ticket).queue).to eq("technical")
|
|
379
|
+
expect(SupportTriage).to have_been_evaluated.once
|
|
380
|
+
end
|
|
381
|
+
```
|
|
382
|
+
|
|
383
|
+
Stubbed values are written the way you would assert on them — `true`, `:technical`,
|
|
384
|
+
`"major"` — and expanded into a response the real API could have returned, probability
|
|
385
|
+
distribution and all. An unstubbed evaluation **raises**, so a new question added to a
|
|
386
|
+
decision surfaces in the suite rather than silently answering `nil`.
|
|
387
|
+
|
|
388
|
+
```ruby
|
|
389
|
+
stub_jevalyn(SupportTriage, urgent: 0.61, ...) # exact probability
|
|
390
|
+
stub_jevalyn(SupportTriage, confidence: 0.4, ...) # exercise the uncertain path
|
|
391
|
+
stub_jevalyn(SupportTriage) { |state| { urgent: state.include?("!") } } # per-state
|
|
392
|
+
forbid_jevalyn(SupportTriage) # assert it is never called
|
|
393
|
+
```
|
|
394
|
+
|
|
395
|
+
`confidence:` also takes a Hash, which is how you test [per-question
|
|
396
|
+
floors](#confidence) — one answer landing under its floor while another clears its own:
|
|
397
|
+
|
|
398
|
+
```ruby
|
|
399
|
+
it "escalates a low-confidence department but keeps the severity" do
|
|
400
|
+
stub_jevalyn(SupportTriage,
|
|
401
|
+
department: :technical, severity: "major",
|
|
402
|
+
confidence: { department: 0.7, severity: 0.65 })
|
|
403
|
+
|
|
404
|
+
result = SupportTriage.evaluate(ticket.body)
|
|
405
|
+
|
|
406
|
+
expect(result).to have_uncertain_questions(:department)
|
|
407
|
+
expect(result).to be_severity_certain
|
|
408
|
+
end
|
|
409
|
+
```
|
|
410
|
+
|
|
411
|
+
`have_uncertain_questions` prints every answer's certainty next to its floor when it
|
|
412
|
+
fails, which is what you need to see when a threshold is set wrong. The per-question
|
|
413
|
+
predicates (`be_department_certain`, `be_severity_certain`) come from `Result` through
|
|
414
|
+
RSpec's own predicate matchers — there is nothing to register.
|
|
415
|
+
|
|
416
|
+
Minitest works the same way via `require "jevalyn/testing/minitest"`.
|
|
417
|
+
|
|
418
|
+
### Cassettes
|
|
419
|
+
|
|
420
|
+
For the handful of tests that should run against real answers, record once and replay:
|
|
421
|
+
|
|
422
|
+
```ruby
|
|
423
|
+
jevalyn_cassette("spec/cassettes/triage.json") do
|
|
424
|
+
result = SupportTriage.evaluate(File.read("spec/fixtures/payout_failure.txt"))
|
|
425
|
+
expect(result.department).to eq(:technical)
|
|
426
|
+
end
|
|
427
|
+
```
|
|
428
|
+
|
|
429
|
+
The first run with a real key records; every run after replays. Requests are keyed by a
|
|
430
|
+
digest of the exact body sent, so changing a rubric misses the cassette rather than
|
|
431
|
+
replaying a stale answer against a question you no longer ask. Commit the file.
|
|
432
|
+
|
|
433
|
+
### Testing against the live model
|
|
434
|
+
|
|
435
|
+
Stubs prove your *code* is right. They cannot prove your *rubric* is right — and a
|
|
436
|
+
rubric is the part that drifts, both when you reword it and when `jev-latest` moves
|
|
437
|
+
under you. Keep a handful of real inputs whose answer you are sure of, tag them
|
|
438
|
+
`:jevalyn_live`, and run them deliberately:
|
|
439
|
+
|
|
440
|
+
```ruby
|
|
441
|
+
it "recognises a payout outage as technical", :jevalyn_live do
|
|
442
|
+
expect(SupportTriage.evaluate(payout_outage_ticket).department).to eq(:technical)
|
|
443
|
+
end
|
|
444
|
+
```
|
|
445
|
+
|
|
446
|
+
## Background evaluation
|
|
447
|
+
|
|
448
|
+
Jev answers in well under a second, so the default advice is to call `evaluate` inline
|
|
449
|
+
and keep the decision in your control flow. That is the whole point of a System One
|
|
450
|
+
model — it is fast enough to be part of the request.
|
|
451
|
+
|
|
452
|
+
Use the queue when the decision is genuinely not on the critical path: backfills, batch
|
|
453
|
+
scoring, or anywhere a third-party outage must not take a request down with it.
|
|
454
|
+
|
|
455
|
+
```ruby
|
|
456
|
+
SupportTriage.evaluate_later(ticket, on: TicketRouter)
|
|
457
|
+
# => TicketRouter.call(result, ticket)
|
|
458
|
+
```
|
|
459
|
+
|
|
460
|
+
The handler is named rather than passed as a block, because a block cannot be
|
|
461
|
+
serialised onto a queue.
|
|
462
|
+
|
|
463
|
+
## Configuration
|
|
464
|
+
|
|
465
|
+
```ruby
|
|
466
|
+
Jevalyn.configure do |c|
|
|
467
|
+
c.api_key = ENV["TYPESAFE_API_KEY"]
|
|
468
|
+
c.default_model = "jev-latest"
|
|
469
|
+
c.timeout = 10
|
|
470
|
+
c.open_timeout = 5
|
|
471
|
+
c.max_retries = 2
|
|
472
|
+
c.mock_mode = Rails.env.test?
|
|
473
|
+
end
|
|
474
|
+
```
|
|
475
|
+
|
|
476
|
+
`jev-latest` is an alias, and an alias moves when TypeSafe ships a release. Every
|
|
477
|
+
`Result` reports the versioned model that actually answered (`result.model`), so log it.
|
|
478
|
+
Once you have tuned thresholds against a version, pin it:
|
|
479
|
+
|
|
480
|
+
```ruby
|
|
481
|
+
class RefundDecision < Jevalyn::Decision
|
|
482
|
+
model "jev-1.13.0"
|
|
483
|
+
end
|
|
484
|
+
```
|
|
485
|
+
|
|
486
|
+
## Errors
|
|
487
|
+
|
|
488
|
+
| Error | When |
|
|
489
|
+
| --- | --- |
|
|
490
|
+
| `Jevalyn::ConfigurationError` | no API key, empty state, a malformed decision |
|
|
491
|
+
| `Jevalyn::InvalidQuestionError` | a rubric that cannot be sent — raised at class definition |
|
|
492
|
+
| `Jevalyn::AuthenticationError` | 401 |
|
|
493
|
+
| `Jevalyn::InvalidRequestError` | 422, with the offending field in `#body` |
|
|
494
|
+
| `Jevalyn::RateLimitError` | 429 — retried automatically |
|
|
495
|
+
| `Jevalyn::OverloadedError` | 529 — retried automatically |
|
|
496
|
+
| `Jevalyn::TimeoutError` | the request did not complete |
|
|
497
|
+
| `Jevalyn::ConnectionError` | TypeSafe was unreachable |
|
|
498
|
+
|
|
499
|
+
429, 529 and 5xx are retried with exponential backoff, honouring the API's
|
|
500
|
+
`retry-after` header when it sends one. Everything else fails immediately, because it
|
|
501
|
+
would fail identically on a retry.
|
|
502
|
+
|
|
503
|
+
**Low confidence is not an error.** It is the model doing its job. `result.uncertain?`
|
|
504
|
+
is a branch in your code, not a rescue.
|
|
505
|
+
|
|
506
|
+
## Generators
|
|
507
|
+
|
|
508
|
+
```console
|
|
509
|
+
$ bin/rails g jevalyn:install
|
|
510
|
+
$ bin/rails g jevalyn:decision SupportTriage urgent:noul department:choice
|
|
511
|
+
$ bin/rails g jevalyn:guardrail ToolCall --allow-above 0.95
|
|
512
|
+
```
|
|
513
|
+
|
|
514
|
+
```console
|
|
515
|
+
$ bin/rails jevalyn:ping # check key, network, available models
|
|
516
|
+
$ bin/rails jevalyn:decisions # list every Decision and Guardrail in the app
|
|
517
|
+
```
|
|
518
|
+
|
|
519
|
+
## Requirements
|
|
520
|
+
|
|
521
|
+
Ruby 3.1+, Rails 7.0+. Jevalyn is Rails-only by design — the generators, the railtie,
|
|
522
|
+
and the ActiveRecord state conventions are the reason it exists rather than a raw
|
|
523
|
+
client.
|
|
524
|
+
|
|
525
|
+
## Relationship to TypeSafe
|
|
526
|
+
|
|
527
|
+
Jevalyn is an unofficial, community-maintained gem. It is not built, endorsed, or
|
|
528
|
+
supported by TypeSafe AI. "Jev" and "TypeSafe" are theirs. For the API itself, the
|
|
529
|
+
canonical reference is [docs.typesafe.ai](https://docs.typesafe.ai).
|
|
530
|
+
|
|
531
|
+
## License
|
|
532
|
+
|
|
533
|
+
MIT. See [LICENSE.txt](LICENSE.txt).
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators/named_base"
|
|
4
|
+
|
|
5
|
+
module Jevalyn
|
|
6
|
+
module Generators
|
|
7
|
+
# rails g jevalyn:decision SupportTriage urgent:noul department:choice severity:score
|
|
8
|
+
#
|
|
9
|
+
# Question arguments are optional; without them you get a commented skeleton of
|
|
10
|
+
# all three types to fill in.
|
|
11
|
+
class DecisionGenerator < ::Rails::Generators::NamedBase
|
|
12
|
+
source_root File.expand_path("templates", __dir__)
|
|
13
|
+
|
|
14
|
+
argument :questions, type: :array, default: [], banner: "name:type name:type"
|
|
15
|
+
|
|
16
|
+
desc "Creates a Jevalyn::Decision in app/decisions, and a spec for it."
|
|
17
|
+
|
|
18
|
+
class_option :threshold, type: :numeric, default: 0.75,
|
|
19
|
+
desc: "Confidence floor for the decision"
|
|
20
|
+
class_option :spec, type: :boolean, default: true,
|
|
21
|
+
desc: "Also generate a spec file"
|
|
22
|
+
|
|
23
|
+
def create_decision
|
|
24
|
+
template "decision.rb.tt", File.join("app/decisions", class_path, "#{file_name}.rb")
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def create_spec
|
|
28
|
+
return unless options[:spec]
|
|
29
|
+
|
|
30
|
+
template "decision_spec.rb.tt", File.join("spec/decisions", class_path, "#{file_name}_spec.rb")
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
# [[name, type], ...] -- defaults to one of each so the file is worth reading.
|
|
36
|
+
def parsed_questions
|
|
37
|
+
return default_questions if questions.empty?
|
|
38
|
+
|
|
39
|
+
questions.map do |argument|
|
|
40
|
+
name, type = argument.split(":", 2)
|
|
41
|
+
type = (type || "noul").to_sym
|
|
42
|
+
|
|
43
|
+
unless Jevalyn::Question::TYPES.include?(type)
|
|
44
|
+
raise Thor::Error,
|
|
45
|
+
"Unknown question type #{type.inspect} for #{name}. " \
|
|
46
|
+
"Jev has three: noul, choice, score."
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
[name, type]
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def default_questions
|
|
54
|
+
[%w[urgent noul].then { |n, t| [n, t.to_sym] }]
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def threshold = options[:threshold]
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|