deeprails 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/README.md +30 -6
- data/lib/deeprails/models/defend_submit_event_params.rb +4 -4
- data/lib/deeprails/models/evaluate_create_params.rb +2 -2
- data/lib/deeprails/models/evaluation.rb +2 -2
- data/lib/deeprails/models/monitor_submit_event_params.rb +2 -2
- data/lib/deeprails/version.rb +1 -1
- data/rbi/deeprails/models/defend_submit_event_params.rbi +6 -6
- data/rbi/deeprails/models/evaluate_create_params.rbi +3 -3
- data/rbi/deeprails/models/evaluation.rbi +3 -3
- data/rbi/deeprails/models/monitor_submit_event_params.rbi +3 -3
- data/rbi/deeprails/resources/defend.rbi +2 -2
- data/rbi/deeprails/resources/evaluate.rbi +1 -1
- data/rbi/deeprails/resources/monitor.rbi +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e7d722f2acb3079695e92cd5455050c9d6b09fb46687b4300fba8eee40f228bf
|
|
4
|
+
data.tar.gz: 5783eb1fdf16fd00387b893187d51d2bf8a157d3f5d51e2318dc525761045da8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 807da14f2454695d4eb036637f6a645985b12aa3e821171aaeec65cd5964ed86086afd822391ea85b8358631c3cd3c73b1b4c262e0e9429a171711e5870552a6
|
|
7
|
+
data.tar.gz: 659273be50dd4dd1f8b43fe5a5d35640359a6b656d51298275c542f3996170bad7ebc7f0dc56b7e914264513eef1595b80fbee4c35e32f2036ae092aea28b26c
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.7.0 (2025-10-22)
|
|
4
|
+
|
|
5
|
+
Full Changelog: [v0.6.0...v0.7.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.6.0...v0.7.0)
|
|
6
|
+
|
|
7
|
+
### Features
|
|
8
|
+
|
|
9
|
+
* **api:** manual updates ([3f3207a](https://github.com/deeprails/deeprails-ruby-sdk/commit/3f3207ad9b0e71ed6911c94cf6850a4d35a8d295))
|
|
10
|
+
|
|
3
11
|
## 0.6.0 (2025-10-22)
|
|
4
12
|
|
|
5
13
|
Full Changelog: [v0.5.0...v0.6.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.5.0...v0.6.0)
|
data/README.md
CHANGED
|
@@ -15,7 +15,7 @@ To use this gem, install via Bundler by adding the following to your application
|
|
|
15
15
|
<!-- x-release-please-start-version -->
|
|
16
16
|
|
|
17
17
|
```ruby
|
|
18
|
-
gem "deeprails", "~> 0.
|
|
18
|
+
gem "deeprails", "~> 0.7.0"
|
|
19
19
|
```
|
|
20
20
|
|
|
21
21
|
<!-- x-release-please-end -->
|
|
@@ -30,7 +30,12 @@ deeprails = Deeprails::Client.new(
|
|
|
30
30
|
api_key: ENV["DEEPRAILS_API_KEY"] # This is the default and can be omitted
|
|
31
31
|
)
|
|
32
32
|
|
|
33
|
-
defend_response = deeprails.defend.create_workflow(
|
|
33
|
+
defend_response = deeprails.defend.create_workflow(
|
|
34
|
+
improvement_action: "fixit",
|
|
35
|
+
name: "Push Alert Workflow",
|
|
36
|
+
type: "custom",
|
|
37
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75}
|
|
38
|
+
)
|
|
34
39
|
|
|
35
40
|
puts(defend_response.workflow_id)
|
|
36
41
|
```
|
|
@@ -41,7 +46,12 @@ When the library is unable to connect to the API, or if the API returns a non-su
|
|
|
41
46
|
|
|
42
47
|
```ruby
|
|
43
48
|
begin
|
|
44
|
-
defend = deeprails.defend.create_workflow(
|
|
49
|
+
defend = deeprails.defend.create_workflow(
|
|
50
|
+
improvement_action: "fixit",
|
|
51
|
+
name: "Push Alert Workflow",
|
|
52
|
+
type: "custom",
|
|
53
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75}
|
|
54
|
+
)
|
|
45
55
|
rescue Deeprails::Errors::APIConnectionError => e
|
|
46
56
|
puts("The server could not be reached")
|
|
47
57
|
puts(e.cause) # an underlying Exception, likely raised within `net/http`
|
|
@@ -88,6 +98,7 @@ deeprails.defend.create_workflow(
|
|
|
88
98
|
improvement_action: "fixit",
|
|
89
99
|
name: "Push Alert Workflow",
|
|
90
100
|
type: "custom",
|
|
101
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75},
|
|
91
102
|
request_options: {max_retries: 5}
|
|
92
103
|
)
|
|
93
104
|
```
|
|
@@ -107,6 +118,7 @@ deeprails.defend.create_workflow(
|
|
|
107
118
|
improvement_action: "fixit",
|
|
108
119
|
name: "Push Alert Workflow",
|
|
109
120
|
type: "custom",
|
|
121
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75},
|
|
110
122
|
request_options: {timeout: 5}
|
|
111
123
|
)
|
|
112
124
|
```
|
|
@@ -143,6 +155,7 @@ defend_response =
|
|
|
143
155
|
improvement_action: "fixit",
|
|
144
156
|
name: "Push Alert Workflow",
|
|
145
157
|
type: "custom",
|
|
158
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75},
|
|
146
159
|
request_options: {
|
|
147
160
|
extra_query: {my_query_parameter: value},
|
|
148
161
|
extra_body: {my_body_parameter: value},
|
|
@@ -188,20 +201,31 @@ This library provides comprehensive [RBI](https://sorbet.org/docs/rbi) definitio
|
|
|
188
201
|
You can provide typesafe request parameters like so:
|
|
189
202
|
|
|
190
203
|
```ruby
|
|
191
|
-
deeprails.defend.create_workflow(
|
|
204
|
+
deeprails.defend.create_workflow(
|
|
205
|
+
improvement_action: "fixit",
|
|
206
|
+
name: "Push Alert Workflow",
|
|
207
|
+
type: "custom",
|
|
208
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75}
|
|
209
|
+
)
|
|
192
210
|
```
|
|
193
211
|
|
|
194
212
|
Or, equivalently:
|
|
195
213
|
|
|
196
214
|
```ruby
|
|
197
215
|
# Hashes work, but are not typesafe:
|
|
198
|
-
deeprails.defend.create_workflow(
|
|
216
|
+
deeprails.defend.create_workflow(
|
|
217
|
+
improvement_action: "fixit",
|
|
218
|
+
name: "Push Alert Workflow",
|
|
219
|
+
type: "custom",
|
|
220
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75}
|
|
221
|
+
)
|
|
199
222
|
|
|
200
223
|
# You can also splat a full Params class:
|
|
201
224
|
params = Deeprails::DefendCreateWorkflowParams.new(
|
|
202
225
|
improvement_action: "fixit",
|
|
203
226
|
name: "Push Alert Workflow",
|
|
204
|
-
type: "custom"
|
|
227
|
+
type: "custom",
|
|
228
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75}
|
|
205
229
|
)
|
|
206
230
|
deeprails.defend.create_workflow(**params)
|
|
207
231
|
```
|
|
@@ -9,8 +9,8 @@ module Deeprails
|
|
|
9
9
|
|
|
10
10
|
# @!attribute model_input
|
|
11
11
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
12
|
-
# contain at least `user_prompt` or `system_prompt` field. For
|
|
13
|
-
#
|
|
12
|
+
# contain at least `user_prompt` or `system_prompt` field. For the
|
|
13
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
14
14
|
#
|
|
15
15
|
# @return [Deeprails::Models::DefendSubmitEventParams::ModelInput]
|
|
16
16
|
required :model_input, -> { Deeprails::DefendSubmitEventParams::ModelInput }
|
|
@@ -79,8 +79,8 @@ module Deeprails
|
|
|
79
79
|
|
|
80
80
|
# @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
|
|
81
81
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
82
|
-
# contain at least `user_prompt` or `system_prompt` field. For
|
|
83
|
-
#
|
|
82
|
+
# contain at least `user_prompt` or `system_prompt` field. For the
|
|
83
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
84
84
|
#
|
|
85
85
|
# @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
86
86
|
#
|
|
@@ -10,7 +10,7 @@ module Deeprails
|
|
|
10
10
|
# @!attribute model_input
|
|
11
11
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
12
12
|
# contain at least `user_prompt` or `system_prompt` field. For
|
|
13
|
-
#
|
|
13
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
14
14
|
#
|
|
15
15
|
# @return [Deeprails::Models::EvaluateCreateParams::ModelInput]
|
|
16
16
|
required :model_input, -> { Deeprails::EvaluateCreateParams::ModelInput }
|
|
@@ -91,7 +91,7 @@ module Deeprails
|
|
|
91
91
|
# @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
|
|
92
92
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
93
93
|
# contain at least `user_prompt` or `system_prompt` field. For
|
|
94
|
-
#
|
|
94
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
95
95
|
#
|
|
96
96
|
# @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
97
97
|
#
|
|
@@ -19,7 +19,7 @@ module Deeprails
|
|
|
19
19
|
# @!attribute model_input
|
|
20
20
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
21
21
|
# contain at least `user_prompt` or `system_prompt` field. For
|
|
22
|
-
#
|
|
22
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
23
23
|
#
|
|
24
24
|
# @return [Deeprails::Models::Evaluation::ModelInput]
|
|
25
25
|
required :model_input, -> { Deeprails::Evaluation::ModelInput }
|
|
@@ -190,7 +190,7 @@ module Deeprails
|
|
|
190
190
|
# @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
|
|
191
191
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
192
192
|
# contain at least `user_prompt` or `system_prompt` field. For
|
|
193
|
-
#
|
|
193
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
194
194
|
#
|
|
195
195
|
# @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
196
196
|
#
|
|
@@ -20,7 +20,7 @@ module Deeprails
|
|
|
20
20
|
# @!attribute model_input
|
|
21
21
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
22
22
|
# contain at least a `user_prompt` or `system_prompt` field. For
|
|
23
|
-
#
|
|
23
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
24
24
|
#
|
|
25
25
|
# @return [Deeprails::Models::MonitorSubmitEventParams::ModelInput]
|
|
26
26
|
required :model_input, -> { Deeprails::MonitorSubmitEventParams::ModelInput }
|
|
@@ -106,7 +106,7 @@ module Deeprails
|
|
|
106
106
|
# @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
|
|
107
107
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
108
108
|
# contain at least a `user_prompt` or `system_prompt` field. For
|
|
109
|
-
#
|
|
109
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
110
110
|
#
|
|
111
111
|
# @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
112
112
|
#
|
data/lib/deeprails/version.rb
CHANGED
|
@@ -15,8 +15,8 @@ module Deeprails
|
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
18
|
-
# contain at least `user_prompt` or `system_prompt` field. For
|
|
19
|
-
#
|
|
18
|
+
# contain at least `user_prompt` or `system_prompt` field. For the
|
|
19
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
20
20
|
sig { returns(Deeprails::DefendSubmitEventParams::ModelInput) }
|
|
21
21
|
attr_reader :model_input
|
|
22
22
|
|
|
@@ -61,8 +61,8 @@ module Deeprails
|
|
|
61
61
|
end
|
|
62
62
|
def self.new(
|
|
63
63
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
64
|
-
# contain at least `user_prompt` or `system_prompt` field. For
|
|
65
|
-
#
|
|
64
|
+
# contain at least `user_prompt` or `system_prompt` field. For the
|
|
65
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
66
66
|
model_input:,
|
|
67
67
|
# Output generated by the LLM to be evaluated.
|
|
68
68
|
model_output:,
|
|
@@ -125,8 +125,8 @@ module Deeprails
|
|
|
125
125
|
attr_writer :user_prompt
|
|
126
126
|
|
|
127
127
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
128
|
-
# contain at least `user_prompt` or `system_prompt` field. For
|
|
129
|
-
#
|
|
128
|
+
# contain at least `user_prompt` or `system_prompt` field. For the
|
|
129
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
130
130
|
sig do
|
|
131
131
|
params(
|
|
132
132
|
ground_truth: String,
|
|
@@ -13,7 +13,7 @@ module Deeprails
|
|
|
13
13
|
|
|
14
14
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
15
15
|
# contain at least `user_prompt` or `system_prompt` field. For
|
|
16
|
-
#
|
|
16
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
17
17
|
sig { returns(Deeprails::EvaluateCreateParams::ModelInput) }
|
|
18
18
|
attr_reader :model_input
|
|
19
19
|
|
|
@@ -86,7 +86,7 @@ module Deeprails
|
|
|
86
86
|
def self.new(
|
|
87
87
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
88
88
|
# contain at least `user_prompt` or `system_prompt` field. For
|
|
89
|
-
#
|
|
89
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
90
90
|
model_input:,
|
|
91
91
|
# Output generated by the LLM to be evaluated.
|
|
92
92
|
model_output:,
|
|
@@ -158,7 +158,7 @@ module Deeprails
|
|
|
158
158
|
|
|
159
159
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
160
160
|
# contain at least `user_prompt` or `system_prompt` field. For
|
|
161
|
-
#
|
|
161
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
162
162
|
sig do
|
|
163
163
|
params(
|
|
164
164
|
ground_truth: String,
|
|
@@ -18,7 +18,7 @@ module Deeprails
|
|
|
18
18
|
|
|
19
19
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
20
20
|
# contain at least `user_prompt` or `system_prompt` field. For
|
|
21
|
-
#
|
|
21
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
22
22
|
sig { returns(Deeprails::Evaluation::ModelInput) }
|
|
23
23
|
attr_reader :model_input
|
|
24
24
|
|
|
@@ -163,7 +163,7 @@ module Deeprails
|
|
|
163
163
|
evaluation_status:,
|
|
164
164
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
165
165
|
# contain at least `user_prompt` or `system_prompt` field. For
|
|
166
|
-
#
|
|
166
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
167
167
|
model_input:,
|
|
168
168
|
# Output generated by the LLM to be evaluated.
|
|
169
169
|
model_output:,
|
|
@@ -299,7 +299,7 @@ module Deeprails
|
|
|
299
299
|
|
|
300
300
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
301
301
|
# contain at least `user_prompt` or `system_prompt` field. For
|
|
302
|
-
#
|
|
302
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
303
303
|
sig do
|
|
304
304
|
params(
|
|
305
305
|
ground_truth: String,
|
|
@@ -29,7 +29,7 @@ module Deeprails
|
|
|
29
29
|
|
|
30
30
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
31
31
|
# contain at least a `user_prompt` or `system_prompt` field. For
|
|
32
|
-
#
|
|
32
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
33
33
|
sig { returns(Deeprails::MonitorSubmitEventParams::ModelInput) }
|
|
34
34
|
attr_reader :model_input
|
|
35
35
|
|
|
@@ -98,7 +98,7 @@ module Deeprails
|
|
|
98
98
|
guardrail_metrics:,
|
|
99
99
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
100
100
|
# contain at least a `user_prompt` or `system_prompt` field. For
|
|
101
|
-
#
|
|
101
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
102
102
|
model_input:,
|
|
103
103
|
# Output generated by the LLM to be evaluated.
|
|
104
104
|
model_output:,
|
|
@@ -217,7 +217,7 @@ module Deeprails
|
|
|
217
217
|
|
|
218
218
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
219
219
|
# contain at least a `user_prompt` or `system_prompt` field. For
|
|
220
|
-
#
|
|
220
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
221
221
|
sig do
|
|
222
222
|
params(
|
|
223
223
|
ground_truth: String,
|
|
@@ -103,8 +103,8 @@ module Deeprails
|
|
|
103
103
|
# Workflow ID associated with this event.
|
|
104
104
|
workflow_id,
|
|
105
105
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
106
|
-
# contain at least `user_prompt` or `system_prompt` field. For
|
|
107
|
-
#
|
|
106
|
+
# contain at least `user_prompt` or `system_prompt` field. For the
|
|
107
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
108
108
|
model_input:,
|
|
109
109
|
# Output generated by the LLM to be evaluated.
|
|
110
110
|
model_output:,
|
|
@@ -22,7 +22,7 @@ module Deeprails
|
|
|
22
22
|
def create(
|
|
23
23
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
24
24
|
# contain at least `user_prompt` or `system_prompt` field. For
|
|
25
|
-
#
|
|
25
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
26
26
|
model_input:,
|
|
27
27
|
# Output generated by the LLM to be evaluated.
|
|
28
28
|
model_output:,
|
|
@@ -92,7 +92,7 @@ module Deeprails
|
|
|
92
92
|
guardrail_metrics:,
|
|
93
93
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
94
94
|
# contain at least a `user_prompt` or `system_prompt` field. For
|
|
95
|
-
#
|
|
95
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
96
96
|
model_input:,
|
|
97
97
|
# Output generated by the LLM to be evaluated.
|
|
98
98
|
model_output:,
|