deeprails 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/README.md +30 -6
- data/lib/deeprails/internal/transport/pooled_net_requester.rb +30 -24
- data/lib/deeprails/models/defend_create_workflow_params.rb +4 -4
- data/lib/deeprails/models/defend_response.rb +4 -4
- data/lib/deeprails/models/defend_submit_event_params.rb +6 -6
- data/lib/deeprails/models/evaluate_create_params.rb +4 -4
- data/lib/deeprails/models/evaluation.rb +4 -4
- data/lib/deeprails/models/monitor_detail_response.rb +150 -0
- data/lib/deeprails/models/monitor_event_response.rb +41 -0
- data/lib/deeprails/models/monitor_response.rb +83 -0
- data/lib/deeprails/models/monitor_submit_event_params.rb +4 -4
- data/lib/deeprails/models.rb +6 -2
- data/lib/deeprails/resources/defend.rb +2 -2
- data/lib/deeprails/resources/monitor.rb +8 -8
- data/lib/deeprails/version.rb +1 -1
- data/lib/deeprails.rb +3 -3
- data/rbi/deeprails/models/defend_create_workflow_params.rbi +6 -6
- data/rbi/deeprails/models/defend_response.rbi +6 -6
- data/rbi/deeprails/models/defend_submit_event_params.rbi +8 -8
- data/rbi/deeprails/models/evaluate_create_params.rbi +6 -6
- data/rbi/deeprails/models/evaluation.rbi +6 -6
- data/rbi/deeprails/models/monitor_detail_response.rbi +251 -0
- data/rbi/deeprails/models/monitor_event_response.rbi +64 -0
- data/rbi/deeprails/models/monitor_response.rbi +142 -0
- data/rbi/deeprails/models/monitor_submit_event_params.rbi +6 -6
- data/rbi/deeprails/models.rbi +6 -2
- data/rbi/deeprails/resources/defend.rbi +4 -4
- data/rbi/deeprails/resources/evaluate.rbi +2 -2
- data/rbi/deeprails/resources/monitor.rbi +6 -6
- data/sig/deeprails/models/defend_create_workflow_params.rbs +5 -5
- data/sig/deeprails/models/defend_response.rbs +5 -5
- data/sig/deeprails/models/monitor_detail_response.rbs +134 -0
- data/sig/deeprails/models/monitor_event_response.rbs +37 -0
- data/sig/deeprails/models/monitor_response.rbs +73 -0
- data/sig/deeprails/models.rbs +6 -2
- data/sig/deeprails/resources/defend.rbs +1 -1
- data/sig/deeprails/resources/monitor.rbs +4 -4
- metadata +11 -11
- data/lib/deeprails/models/api_response.rb +0 -116
- data/lib/deeprails/models/monitor_retrieve_response.rb +0 -183
- data/lib/deeprails/models/monitor_submit_event_response.rb +0 -74
- data/rbi/deeprails/models/api_response.rbi +0 -201
- data/rbi/deeprails/models/monitor_retrieve_response.rbi +0 -333
- data/rbi/deeprails/models/monitor_submit_event_response.rbi +0 -131
- data/sig/deeprails/models/api_response.rbs +0 -100
- data/sig/deeprails/models/monitor_retrieve_response.rbs +0 -167
- data/sig/deeprails/models/monitor_submit_event_response.rbs +0 -70
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4059b07f012b97e31dfe283afca63de3e15fbcd8b4707a83b1a89bb2c064ac73
|
|
4
|
+
data.tar.gz: a15f42872e3ea1ed96fa9fafc507f20725532dc9a65e09f573863d9030b38879
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1df27d336394dbfa922d7ba6e410b7d37918803fb3e0c85706774bac03cdad2ebc48491a8c988306129e72f66b87054866d7d76d0a760312f0ba4b5c84d93a4e
|
|
7
|
+
data.tar.gz: a07104cbb7fec46daa6cd991194c370aea807ce7d4e2732c0dd26eb485952ff4962b33073ff2411769612bdb9661ab2d911105a090766ae954d8a802620d1f18
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.8.0 (2025-10-24)
|
|
4
|
+
|
|
5
|
+
Full Changelog: [v0.7.0...v0.8.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.7.0...v0.8.0)
|
|
6
|
+
|
|
7
|
+
### Features
|
|
8
|
+
|
|
9
|
+
* **api:** remove apiresponse from monitor ([c26404c](https://github.com/deeprails/deeprails-ruby-sdk/commit/c26404cf6e750823780141362084731adf50b089))
|
|
10
|
+
* handle thread interrupts in the core HTTP client ([7c7e928](https://github.com/deeprails/deeprails-ruby-sdk/commit/7c7e928f1ec5bb401bce8200ec186e8c080de890))
|
|
11
|
+
|
|
12
|
+
## 0.7.0 (2025-10-22)
|
|
13
|
+
|
|
14
|
+
Full Changelog: [v0.6.0...v0.7.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.6.0...v0.7.0)
|
|
15
|
+
|
|
16
|
+
### Features
|
|
17
|
+
|
|
18
|
+
* **api:** manual updates ([3f3207a](https://github.com/deeprails/deeprails-ruby-sdk/commit/3f3207ad9b0e71ed6911c94cf6850a4d35a8d295))
|
|
19
|
+
|
|
3
20
|
## 0.6.0 (2025-10-22)
|
|
4
21
|
|
|
5
22
|
Full Changelog: [v0.5.0...v0.6.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.5.0...v0.6.0)
|
data/README.md
CHANGED
|
@@ -15,7 +15,7 @@ To use this gem, install via Bundler by adding the following to your application
|
|
|
15
15
|
<!-- x-release-please-start-version -->
|
|
16
16
|
|
|
17
17
|
```ruby
|
|
18
|
-
gem "deeprails", "~> 0.
|
|
18
|
+
gem "deeprails", "~> 0.8.0"
|
|
19
19
|
```
|
|
20
20
|
|
|
21
21
|
<!-- x-release-please-end -->
|
|
@@ -30,7 +30,12 @@ deeprails = Deeprails::Client.new(
|
|
|
30
30
|
api_key: ENV["DEEPRAILS_API_KEY"] # This is the default and can be omitted
|
|
31
31
|
)
|
|
32
32
|
|
|
33
|
-
defend_response = deeprails.defend.create_workflow(
|
|
33
|
+
defend_response = deeprails.defend.create_workflow(
|
|
34
|
+
improvement_action: "fixit",
|
|
35
|
+
name: "Push Alert Workflow",
|
|
36
|
+
type: "custom",
|
|
37
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75}
|
|
38
|
+
)
|
|
34
39
|
|
|
35
40
|
puts(defend_response.workflow_id)
|
|
36
41
|
```
|
|
@@ -41,7 +46,12 @@ When the library is unable to connect to the API, or if the API returns a non-su
|
|
|
41
46
|
|
|
42
47
|
```ruby
|
|
43
48
|
begin
|
|
44
|
-
defend = deeprails.defend.create_workflow(
|
|
49
|
+
defend = deeprails.defend.create_workflow(
|
|
50
|
+
improvement_action: "fixit",
|
|
51
|
+
name: "Push Alert Workflow",
|
|
52
|
+
type: "custom",
|
|
53
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75}
|
|
54
|
+
)
|
|
45
55
|
rescue Deeprails::Errors::APIConnectionError => e
|
|
46
56
|
puts("The server could not be reached")
|
|
47
57
|
puts(e.cause) # an underlying Exception, likely raised within `net/http`
|
|
@@ -88,6 +98,7 @@ deeprails.defend.create_workflow(
|
|
|
88
98
|
improvement_action: "fixit",
|
|
89
99
|
name: "Push Alert Workflow",
|
|
90
100
|
type: "custom",
|
|
101
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75},
|
|
91
102
|
request_options: {max_retries: 5}
|
|
92
103
|
)
|
|
93
104
|
```
|
|
@@ -107,6 +118,7 @@ deeprails.defend.create_workflow(
|
|
|
107
118
|
improvement_action: "fixit",
|
|
108
119
|
name: "Push Alert Workflow",
|
|
109
120
|
type: "custom",
|
|
121
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75},
|
|
110
122
|
request_options: {timeout: 5}
|
|
111
123
|
)
|
|
112
124
|
```
|
|
@@ -143,6 +155,7 @@ defend_response =
|
|
|
143
155
|
improvement_action: "fixit",
|
|
144
156
|
name: "Push Alert Workflow",
|
|
145
157
|
type: "custom",
|
|
158
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75},
|
|
146
159
|
request_options: {
|
|
147
160
|
extra_query: {my_query_parameter: value},
|
|
148
161
|
extra_body: {my_body_parameter: value},
|
|
@@ -188,20 +201,31 @@ This library provides comprehensive [RBI](https://sorbet.org/docs/rbi) definitio
|
|
|
188
201
|
You can provide typesafe request parameters like so:
|
|
189
202
|
|
|
190
203
|
```ruby
|
|
191
|
-
deeprails.defend.create_workflow(
|
|
204
|
+
deeprails.defend.create_workflow(
|
|
205
|
+
improvement_action: "fixit",
|
|
206
|
+
name: "Push Alert Workflow",
|
|
207
|
+
type: "custom",
|
|
208
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75}
|
|
209
|
+
)
|
|
192
210
|
```
|
|
193
211
|
|
|
194
212
|
Or, equivalently:
|
|
195
213
|
|
|
196
214
|
```ruby
|
|
197
215
|
# Hashes work, but are not typesafe:
|
|
198
|
-
deeprails.defend.create_workflow(
|
|
216
|
+
deeprails.defend.create_workflow(
|
|
217
|
+
improvement_action: "fixit",
|
|
218
|
+
name: "Push Alert Workflow",
|
|
219
|
+
type: "custom",
|
|
220
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75}
|
|
221
|
+
)
|
|
199
222
|
|
|
200
223
|
# You can also splat a full Params class:
|
|
201
224
|
params = Deeprails::DefendCreateWorkflowParams.new(
|
|
202
225
|
improvement_action: "fixit",
|
|
203
226
|
name: "Push Alert Workflow",
|
|
204
|
-
type: "custom"
|
|
227
|
+
type: "custom",
|
|
228
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75}
|
|
205
229
|
)
|
|
206
230
|
deeprails.defend.create_workflow(**params)
|
|
207
231
|
```
|
|
@@ -128,40 +128,48 @@ module Deeprails
|
|
|
128
128
|
url, deadline = request.fetch_values(:url, :deadline)
|
|
129
129
|
|
|
130
130
|
req = nil
|
|
131
|
-
eof = false
|
|
132
131
|
finished = false
|
|
133
|
-
closing = nil
|
|
134
132
|
|
|
135
133
|
# rubocop:disable Metrics/BlockLength
|
|
136
134
|
enum = Enumerator.new do |y|
|
|
137
135
|
next if finished
|
|
138
136
|
|
|
139
137
|
with_pool(url, deadline: deadline) do |conn|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
conn.start
|
|
148
|
-
end
|
|
138
|
+
eof = false
|
|
139
|
+
closing = nil
|
|
140
|
+
::Thread.handle_interrupt(Object => :never) do
|
|
141
|
+
::Thread.handle_interrupt(Object => :immediate) do
|
|
142
|
+
req, closing = self.class.build_request(request) do
|
|
143
|
+
self.class.calibrate_socket_timeout(conn, deadline)
|
|
144
|
+
end
|
|
149
145
|
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
rsp.read_body do |bytes|
|
|
156
|
-
y << bytes.force_encoding(Encoding::BINARY)
|
|
157
|
-
break if finished
|
|
146
|
+
self.class.calibrate_socket_timeout(conn, deadline)
|
|
147
|
+
unless conn.started?
|
|
148
|
+
conn.keep_alive_timeout = self.class::KEEP_ALIVE_TIMEOUT
|
|
149
|
+
conn.start
|
|
150
|
+
end
|
|
158
151
|
|
|
159
152
|
self.class.calibrate_socket_timeout(conn, deadline)
|
|
153
|
+
conn.request(req) do |rsp|
|
|
154
|
+
y << [req, rsp]
|
|
155
|
+
break if finished
|
|
156
|
+
|
|
157
|
+
rsp.read_body do |bytes|
|
|
158
|
+
y << bytes.force_encoding(Encoding::BINARY)
|
|
159
|
+
break if finished
|
|
160
|
+
|
|
161
|
+
self.class.calibrate_socket_timeout(conn, deadline)
|
|
162
|
+
end
|
|
163
|
+
eof = true
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
ensure
|
|
167
|
+
begin
|
|
168
|
+
conn.finish if !eof && conn&.started?
|
|
169
|
+
ensure
|
|
170
|
+
closing&.call
|
|
160
171
|
end
|
|
161
|
-
eof = true
|
|
162
172
|
end
|
|
163
|
-
ensure
|
|
164
|
-
conn.finish if !eof && conn&.started?
|
|
165
173
|
end
|
|
166
174
|
rescue Timeout::Error
|
|
167
175
|
raise Deeprails::Errors::APITimeoutError.new(url: url, request: req)
|
|
@@ -174,8 +182,6 @@ module Deeprails
|
|
|
174
182
|
body = Deeprails::Internal::Util.fused_enum(enum, external: true) do
|
|
175
183
|
finished = true
|
|
176
184
|
loop { enum.next }
|
|
177
|
-
ensure
|
|
178
|
-
closing&.call
|
|
179
185
|
end
|
|
180
186
|
[Integer(response.code), response, body]
|
|
181
187
|
end
|
|
@@ -57,14 +57,14 @@ module Deeprails
|
|
|
57
57
|
# @return [String, nil]
|
|
58
58
|
optional :description, String
|
|
59
59
|
|
|
60
|
-
# @!attribute
|
|
60
|
+
# @!attribute max_improvement_attempts
|
|
61
61
|
# Max. number of improvement action retries until a given event passes the
|
|
62
62
|
# guardrails. Defaults to 10.
|
|
63
63
|
#
|
|
64
64
|
# @return [Integer, nil]
|
|
65
|
-
optional :
|
|
65
|
+
optional :max_improvement_attempts, Integer
|
|
66
66
|
|
|
67
|
-
# @!method initialize(improvement_action:, name:, type:, automatic_hallucination_tolerance_levels: nil, custom_hallucination_threshold_values: nil, description: nil,
|
|
67
|
+
# @!method initialize(improvement_action:, name:, type:, automatic_hallucination_tolerance_levels: nil, custom_hallucination_threshold_values: nil, description: nil, max_improvement_attempts: nil, request_options: {})
|
|
68
68
|
# Some parameter documentations has been truncated, see
|
|
69
69
|
# {Deeprails::Models::DefendCreateWorkflowParams} for more details.
|
|
70
70
|
#
|
|
@@ -80,7 +80,7 @@ module Deeprails
|
|
|
80
80
|
#
|
|
81
81
|
# @param description [String] Description for the workflow.
|
|
82
82
|
#
|
|
83
|
-
# @param
|
|
83
|
+
# @param max_improvement_attempts [Integer] Max. number of improvement action retries until a given event passes the guardra
|
|
84
84
|
#
|
|
85
85
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
86
86
|
|
|
@@ -38,12 +38,12 @@ module Deeprails
|
|
|
38
38
|
# @return [Symbol, Deeprails::Models::DefendResponse::ImprovementAction, nil]
|
|
39
39
|
optional :improvement_action, enum: -> { Deeprails::DefendResponse::ImprovementAction }
|
|
40
40
|
|
|
41
|
-
# @!attribute
|
|
41
|
+
# @!attribute max_improvement_attempts
|
|
42
42
|
# Max. number of improvement action retries until a given event passes the
|
|
43
43
|
# guardrails.
|
|
44
44
|
#
|
|
45
45
|
# @return [Integer, nil]
|
|
46
|
-
optional :
|
|
46
|
+
optional :max_improvement_attempts, Integer
|
|
47
47
|
|
|
48
48
|
# @!attribute modified_at
|
|
49
49
|
# The most recent time the workflow was modified in UTC.
|
|
@@ -64,7 +64,7 @@ module Deeprails
|
|
|
64
64
|
# @return [Float, nil]
|
|
65
65
|
optional :success_rate, Float
|
|
66
66
|
|
|
67
|
-
# @!method initialize(name:, workflow_id:, created_at: nil, description: nil, improvement_action: nil,
|
|
67
|
+
# @!method initialize(name:, workflow_id:, created_at: nil, description: nil, improvement_action: nil, max_improvement_attempts: nil, modified_at: nil, status: nil, success_rate: nil)
|
|
68
68
|
# Some parameter documentations has been truncated, see
|
|
69
69
|
# {Deeprails::Models::DefendResponse} for more details.
|
|
70
70
|
#
|
|
@@ -78,7 +78,7 @@ module Deeprails
|
|
|
78
78
|
#
|
|
79
79
|
# @param improvement_action [Symbol, Deeprails::Models::DefendResponse::ImprovementAction] The action used to improve outputs that fail one or more guardrail metrics for t
|
|
80
80
|
#
|
|
81
|
-
# @param
|
|
81
|
+
# @param max_improvement_attempts [Integer] Max. number of improvement action retries until a given event passes the guardra
|
|
82
82
|
#
|
|
83
83
|
# @param modified_at [Time] The most recent time the workflow was modified in UTC.
|
|
84
84
|
#
|
|
@@ -9,8 +9,8 @@ module Deeprails
|
|
|
9
9
|
|
|
10
10
|
# @!attribute model_input
|
|
11
11
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
12
|
-
# contain at least `user_prompt` or `system_prompt` field. For
|
|
13
|
-
#
|
|
12
|
+
# contain at least a `user_prompt` field or a `system_prompt` field. For the
|
|
13
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
14
14
|
#
|
|
15
15
|
# @return [Deeprails::Models::DefendSubmitEventParams::ModelInput]
|
|
16
16
|
required :model_input, -> { Deeprails::DefendSubmitEventParams::ModelInput }
|
|
@@ -60,7 +60,7 @@ module Deeprails
|
|
|
60
60
|
|
|
61
61
|
class ModelInput < Deeprails::Internal::Type::BaseModel
|
|
62
62
|
# @!attribute ground_truth
|
|
63
|
-
# The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
63
|
+
# The ground truth for evaluating the Ground Truth Adherence guardrail.
|
|
64
64
|
#
|
|
65
65
|
# @return [String, nil]
|
|
66
66
|
optional :ground_truth, String
|
|
@@ -79,10 +79,10 @@ module Deeprails
|
|
|
79
79
|
|
|
80
80
|
# @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
|
|
81
81
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
82
|
-
# contain at least `user_prompt` or `system_prompt` field. For
|
|
83
|
-
#
|
|
82
|
+
# contain at least a `user_prompt` field or a `system_prompt` field. For the
|
|
83
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
84
84
|
#
|
|
85
|
-
# @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
85
|
+
# @param ground_truth [String] The ground truth for evaluating the Ground Truth Adherence guardrail.
|
|
86
86
|
#
|
|
87
87
|
# @param system_prompt [String] The system prompt used to generate the output.
|
|
88
88
|
#
|
|
@@ -9,8 +9,8 @@ module Deeprails
|
|
|
9
9
|
|
|
10
10
|
# @!attribute model_input
|
|
11
11
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
12
|
-
# contain at least `user_prompt` or `system_prompt` field. For
|
|
13
|
-
#
|
|
12
|
+
# contain at least a `user_prompt` field or a `system_prompt` field. For
|
|
13
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
14
14
|
#
|
|
15
15
|
# @return [Deeprails::Models::EvaluateCreateParams::ModelInput]
|
|
16
16
|
required :model_input, -> { Deeprails::EvaluateCreateParams::ModelInput }
|
|
@@ -90,8 +90,8 @@ module Deeprails
|
|
|
90
90
|
|
|
91
91
|
# @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
|
|
92
92
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
93
|
-
# contain at least `user_prompt` or `system_prompt` field. For
|
|
94
|
-
#
|
|
93
|
+
# contain at least a `user_prompt` field or a `system_prompt` field. For
|
|
94
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
95
95
|
#
|
|
96
96
|
# @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
97
97
|
#
|
|
@@ -18,8 +18,8 @@ module Deeprails
|
|
|
18
18
|
|
|
19
19
|
# @!attribute model_input
|
|
20
20
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
21
|
-
# contain at least `user_prompt` or `system_prompt` field. For
|
|
22
|
-
#
|
|
21
|
+
# contain at least a `user_prompt` field or a `system_prompt` field. For
|
|
22
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
23
23
|
#
|
|
24
24
|
# @return [Deeprails::Models::Evaluation::ModelInput]
|
|
25
25
|
required :model_input, -> { Deeprails::Evaluation::ModelInput }
|
|
@@ -189,8 +189,8 @@ module Deeprails
|
|
|
189
189
|
|
|
190
190
|
# @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
|
|
191
191
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
192
|
-
# contain at least `user_prompt` or `system_prompt` field. For
|
|
193
|
-
#
|
|
192
|
+
# contain at least a `user_prompt` field or a `system_prompt` field. For
|
|
193
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
194
194
|
#
|
|
195
195
|
# @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
196
196
|
#
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Deeprails
|
|
4
|
+
module Models
|
|
5
|
+
# @see Deeprails::Resources::Monitor#retrieve
|
|
6
|
+
class MonitorDetailResponse < Deeprails::Internal::Type::BaseModel
|
|
7
|
+
# @!attribute monitor_id
|
|
8
|
+
# A unique monitor ID.
|
|
9
|
+
#
|
|
10
|
+
# @return [String]
|
|
11
|
+
required :monitor_id, String
|
|
12
|
+
|
|
13
|
+
# @!attribute monitor_status
|
|
14
|
+
# Status of the monitor. Can be `active` or `inactive`. Inactive monitors no
|
|
15
|
+
# longer record and evaluate events.
|
|
16
|
+
#
|
|
17
|
+
# @return [Symbol, Deeprails::Models::MonitorDetailResponse::MonitorStatus]
|
|
18
|
+
required :monitor_status, enum: -> { Deeprails::MonitorDetailResponse::MonitorStatus }
|
|
19
|
+
|
|
20
|
+
# @!attribute name
|
|
21
|
+
# Name of this monitor.
|
|
22
|
+
#
|
|
23
|
+
# @return [String]
|
|
24
|
+
required :name, String
|
|
25
|
+
|
|
26
|
+
# @!attribute created_at
|
|
27
|
+
# The time the monitor was created in UTC.
|
|
28
|
+
#
|
|
29
|
+
# @return [Time, nil]
|
|
30
|
+
optional :created_at, Time
|
|
31
|
+
|
|
32
|
+
# @!attribute description
|
|
33
|
+
# Description of this monitor.
|
|
34
|
+
#
|
|
35
|
+
# @return [String, nil]
|
|
36
|
+
optional :description, String
|
|
37
|
+
|
|
38
|
+
# @!attribute evaluations
|
|
39
|
+
# An array of all evaluations performed by this monitor. Each one corresponds to a
|
|
40
|
+
# separate monitor event.
|
|
41
|
+
#
|
|
42
|
+
# @return [Array<Deeprails::Models::Evaluation>, nil]
|
|
43
|
+
optional :evaluations, -> { Deeprails::Internal::Type::ArrayOf[Deeprails::Evaluation] }
|
|
44
|
+
|
|
45
|
+
# @!attribute stats
|
|
46
|
+
# Contains five fields used for stats of this monitor: total evaluations,
|
|
47
|
+
# completed evaluations, failed evaluations, queued evaluations, and in progress
|
|
48
|
+
# evaluations.
|
|
49
|
+
#
|
|
50
|
+
# @return [Deeprails::Models::MonitorDetailResponse::Stats, nil]
|
|
51
|
+
optional :stats, -> { Deeprails::MonitorDetailResponse::Stats }
|
|
52
|
+
|
|
53
|
+
# @!attribute updated_at
|
|
54
|
+
# The most recent time the monitor was modified in UTC.
|
|
55
|
+
#
|
|
56
|
+
# @return [Time, nil]
|
|
57
|
+
optional :updated_at, Time
|
|
58
|
+
|
|
59
|
+
# @!attribute user_id
|
|
60
|
+
# User ID of the user who created the monitor.
|
|
61
|
+
#
|
|
62
|
+
# @return [String, nil]
|
|
63
|
+
optional :user_id, String
|
|
64
|
+
|
|
65
|
+
# @!method initialize(monitor_id:, monitor_status:, name:, created_at: nil, description: nil, evaluations: nil, stats: nil, updated_at: nil, user_id: nil)
|
|
66
|
+
# Some parameter documentations has been truncated, see
|
|
67
|
+
# {Deeprails::Models::MonitorDetailResponse} for more details.
|
|
68
|
+
#
|
|
69
|
+
# @param monitor_id [String] A unique monitor ID.
|
|
70
|
+
#
|
|
71
|
+
# @param monitor_status [Symbol, Deeprails::Models::MonitorDetailResponse::MonitorStatus] Status of the monitor. Can be `active` or `inactive`. Inactive monitors no lon
|
|
72
|
+
#
|
|
73
|
+
# @param name [String] Name of this monitor.
|
|
74
|
+
#
|
|
75
|
+
# @param created_at [Time] The time the monitor was created in UTC.
|
|
76
|
+
#
|
|
77
|
+
# @param description [String] Description of this monitor.
|
|
78
|
+
#
|
|
79
|
+
# @param evaluations [Array<Deeprails::Models::Evaluation>] An array of all evaluations performed by this monitor. Each one corresponds to
|
|
80
|
+
#
|
|
81
|
+
# @param stats [Deeprails::Models::MonitorDetailResponse::Stats] Contains five fields used for stats of this monitor: total evaluations, complete
|
|
82
|
+
#
|
|
83
|
+
# @param updated_at [Time] The most recent time the monitor was modified in UTC.
|
|
84
|
+
#
|
|
85
|
+
# @param user_id [String] User ID of the user who created the monitor.
|
|
86
|
+
|
|
87
|
+
# Status of the monitor. Can be `active` or `inactive`. Inactive monitors no
|
|
88
|
+
# longer record and evaluate events.
|
|
89
|
+
#
|
|
90
|
+
# @see Deeprails::Models::MonitorDetailResponse#monitor_status
|
|
91
|
+
module MonitorStatus
|
|
92
|
+
extend Deeprails::Internal::Type::Enum
|
|
93
|
+
|
|
94
|
+
ACTIVE = :active
|
|
95
|
+
INACTIVE = :inactive
|
|
96
|
+
|
|
97
|
+
# @!method self.values
|
|
98
|
+
# @return [Array<Symbol>]
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# @see Deeprails::Models::MonitorDetailResponse#stats
|
|
102
|
+
class Stats < Deeprails::Internal::Type::BaseModel
|
|
103
|
+
# @!attribute completed_evaluations
|
|
104
|
+
# Number of evaluations that completed successfully.
|
|
105
|
+
#
|
|
106
|
+
# @return [Integer, nil]
|
|
107
|
+
optional :completed_evaluations, Integer
|
|
108
|
+
|
|
109
|
+
# @!attribute failed_evaluations
|
|
110
|
+
# Number of evaluations that failed.
|
|
111
|
+
#
|
|
112
|
+
# @return [Integer, nil]
|
|
113
|
+
optional :failed_evaluations, Integer
|
|
114
|
+
|
|
115
|
+
# @!attribute in_progress_evaluations
|
|
116
|
+
# Number of evaluations currently in progress.
|
|
117
|
+
#
|
|
118
|
+
# @return [Integer, nil]
|
|
119
|
+
optional :in_progress_evaluations, Integer
|
|
120
|
+
|
|
121
|
+
# @!attribute queued_evaluations
|
|
122
|
+
# Number of evaluations currently queued.
|
|
123
|
+
#
|
|
124
|
+
# @return [Integer, nil]
|
|
125
|
+
optional :queued_evaluations, Integer
|
|
126
|
+
|
|
127
|
+
# @!attribute total_evaluations
|
|
128
|
+
# Total number of evaluations performed by this monitor.
|
|
129
|
+
#
|
|
130
|
+
# @return [Integer, nil]
|
|
131
|
+
optional :total_evaluations, Integer
|
|
132
|
+
|
|
133
|
+
# @!method initialize(completed_evaluations: nil, failed_evaluations: nil, in_progress_evaluations: nil, queued_evaluations: nil, total_evaluations: nil)
|
|
134
|
+
# Contains five fields used for stats of this monitor: total evaluations,
|
|
135
|
+
# completed evaluations, failed evaluations, queued evaluations, and in progress
|
|
136
|
+
# evaluations.
|
|
137
|
+
#
|
|
138
|
+
# @param completed_evaluations [Integer] Number of evaluations that completed successfully.
|
|
139
|
+
#
|
|
140
|
+
# @param failed_evaluations [Integer] Number of evaluations that failed.
|
|
141
|
+
#
|
|
142
|
+
# @param in_progress_evaluations [Integer] Number of evaluations currently in progress.
|
|
143
|
+
#
|
|
144
|
+
# @param queued_evaluations [Integer] Number of evaluations currently queued.
|
|
145
|
+
#
|
|
146
|
+
# @param total_evaluations [Integer] Total number of evaluations performed by this monitor.
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Deeprails
|
|
4
|
+
module Models
|
|
5
|
+
# @see Deeprails::Resources::Monitor#submit_event
|
|
6
|
+
class MonitorEventResponse < Deeprails::Internal::Type::BaseModel
|
|
7
|
+
# @!attribute evaluation_id
|
|
8
|
+
# A unique evaluation ID associated with this event.
|
|
9
|
+
#
|
|
10
|
+
# @return [String]
|
|
11
|
+
required :evaluation_id, String
|
|
12
|
+
|
|
13
|
+
# @!attribute event_id
|
|
14
|
+
# A unique monitor event ID.
|
|
15
|
+
#
|
|
16
|
+
# @return [String]
|
|
17
|
+
required :event_id, String
|
|
18
|
+
|
|
19
|
+
# @!attribute monitor_id
|
|
20
|
+
# Monitor ID associated with this event.
|
|
21
|
+
#
|
|
22
|
+
# @return [String]
|
|
23
|
+
required :monitor_id, String
|
|
24
|
+
|
|
25
|
+
# @!attribute created_at
|
|
26
|
+
# The time the monitor event was created in UTC.
|
|
27
|
+
#
|
|
28
|
+
# @return [Time, nil]
|
|
29
|
+
optional :created_at, Time
|
|
30
|
+
|
|
31
|
+
# @!method initialize(evaluation_id:, event_id:, monitor_id:, created_at: nil)
|
|
32
|
+
# @param evaluation_id [String] A unique evaluation ID associated with this event.
|
|
33
|
+
#
|
|
34
|
+
# @param event_id [String] A unique monitor event ID.
|
|
35
|
+
#
|
|
36
|
+
# @param monitor_id [String] Monitor ID associated with this event.
|
|
37
|
+
#
|
|
38
|
+
# @param created_at [Time] The time the monitor event was created in UTC.
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Deeprails
|
|
4
|
+
module Models
|
|
5
|
+
# @see Deeprails::Resources::Monitor#create
|
|
6
|
+
class MonitorResponse < Deeprails::Internal::Type::BaseModel
|
|
7
|
+
# @!attribute monitor_id
|
|
8
|
+
# A unique monitor ID.
|
|
9
|
+
#
|
|
10
|
+
# @return [String]
|
|
11
|
+
required :monitor_id, String
|
|
12
|
+
|
|
13
|
+
# @!attribute name
|
|
14
|
+
# Name of the monitor.
|
|
15
|
+
#
|
|
16
|
+
# @return [String]
|
|
17
|
+
required :name, String
|
|
18
|
+
|
|
19
|
+
# @!attribute created_at
|
|
20
|
+
# The time the monitor was created in UTC.
|
|
21
|
+
#
|
|
22
|
+
# @return [Time, nil]
|
|
23
|
+
optional :created_at, Time
|
|
24
|
+
|
|
25
|
+
# @!attribute description
|
|
26
|
+
# Description of the monitor.
|
|
27
|
+
#
|
|
28
|
+
# @return [String, nil]
|
|
29
|
+
optional :description, String
|
|
30
|
+
|
|
31
|
+
# @!attribute monitor_status
|
|
32
|
+
# Status of the monitor. Can be `active` or `inactive`. Inactive monitors no
|
|
33
|
+
# longer record and evaluate events.
|
|
34
|
+
#
|
|
35
|
+
# @return [Symbol, Deeprails::Models::MonitorResponse::MonitorStatus, nil]
|
|
36
|
+
optional :monitor_status, enum: -> { Deeprails::MonitorResponse::MonitorStatus }
|
|
37
|
+
|
|
38
|
+
# @!attribute updated_at
|
|
39
|
+
# The most recent time the monitor was modified in UTC.
|
|
40
|
+
#
|
|
41
|
+
# @return [Time, nil]
|
|
42
|
+
optional :updated_at, Time
|
|
43
|
+
|
|
44
|
+
# @!attribute user_id
|
|
45
|
+
# User ID of the user who created the monitor.
|
|
46
|
+
#
|
|
47
|
+
# @return [String, nil]
|
|
48
|
+
optional :user_id, String
|
|
49
|
+
|
|
50
|
+
# @!method initialize(monitor_id:, name:, created_at: nil, description: nil, monitor_status: nil, updated_at: nil, user_id: nil)
|
|
51
|
+
# Some parameter documentations has been truncated, see
|
|
52
|
+
# {Deeprails::Models::MonitorResponse} for more details.
|
|
53
|
+
#
|
|
54
|
+
# @param monitor_id [String] A unique monitor ID.
|
|
55
|
+
#
|
|
56
|
+
# @param name [String] Name of the monitor.
|
|
57
|
+
#
|
|
58
|
+
# @param created_at [Time] The time the monitor was created in UTC.
|
|
59
|
+
#
|
|
60
|
+
# @param description [String] Description of the monitor.
|
|
61
|
+
#
|
|
62
|
+
# @param monitor_status [Symbol, Deeprails::Models::MonitorResponse::MonitorStatus] Status of the monitor. Can be `active` or `inactive`. Inactive monitors no lon
|
|
63
|
+
#
|
|
64
|
+
# @param updated_at [Time] The most recent time the monitor was modified in UTC.
|
|
65
|
+
#
|
|
66
|
+
# @param user_id [String] User ID of the user who created the monitor.
|
|
67
|
+
|
|
68
|
+
# Status of the monitor. Can be `active` or `inactive`. Inactive monitors no
|
|
69
|
+
# longer record and evaluate events.
|
|
70
|
+
#
|
|
71
|
+
# @see Deeprails::Models::MonitorResponse#monitor_status
|
|
72
|
+
module MonitorStatus
|
|
73
|
+
extend Deeprails::Internal::Type::Enum
|
|
74
|
+
|
|
75
|
+
ACTIVE = :active
|
|
76
|
+
INACTIVE = :inactive
|
|
77
|
+
|
|
78
|
+
# @!method self.values
|
|
79
|
+
# @return [Array<Symbol>]
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
@@ -19,8 +19,8 @@ module Deeprails
|
|
|
19
19
|
|
|
20
20
|
# @!attribute model_input
|
|
21
21
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
22
|
-
# contain at least a `user_prompt` or `system_prompt` field. For
|
|
23
|
-
#
|
|
22
|
+
# contain at least a `user_prompt` field or a `system_prompt` field. For
|
|
23
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
24
24
|
#
|
|
25
25
|
# @return [Deeprails::Models::MonitorSubmitEventParams::ModelInput]
|
|
26
26
|
required :model_input, -> { Deeprails::MonitorSubmitEventParams::ModelInput }
|
|
@@ -105,8 +105,8 @@ module Deeprails
|
|
|
105
105
|
|
|
106
106
|
# @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
|
|
107
107
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
108
|
-
# contain at least a `user_prompt` or `system_prompt` field. For
|
|
109
|
-
#
|
|
108
|
+
# contain at least a `user_prompt` field or a `system_prompt` field. For
|
|
109
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
110
110
|
#
|
|
111
111
|
# @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
112
112
|
#
|
data/lib/deeprails/models.rb
CHANGED
|
@@ -39,8 +39,6 @@ module Deeprails
|
|
|
39
39
|
mod.define_sorbet_constant!(const) { T.type_alias { mod.to_sorbet_type } }
|
|
40
40
|
end
|
|
41
41
|
|
|
42
|
-
APIResponse = Deeprails::Models::APIResponse
|
|
43
|
-
|
|
44
42
|
DefendCreateWorkflowParams = Deeprails::Models::DefendCreateWorkflowParams
|
|
45
43
|
|
|
46
44
|
DefendResponse = Deeprails::Models::DefendResponse
|
|
@@ -61,6 +59,12 @@ module Deeprails
|
|
|
61
59
|
|
|
62
60
|
MonitorCreateParams = Deeprails::Models::MonitorCreateParams
|
|
63
61
|
|
|
62
|
+
MonitorDetailResponse = Deeprails::Models::MonitorDetailResponse
|
|
63
|
+
|
|
64
|
+
MonitorEventResponse = Deeprails::Models::MonitorEventResponse
|
|
65
|
+
|
|
66
|
+
MonitorResponse = Deeprails::Models::MonitorResponse
|
|
67
|
+
|
|
64
68
|
MonitorRetrieveParams = Deeprails::Models::MonitorRetrieveParams
|
|
65
69
|
|
|
66
70
|
MonitorSubmitEventParams = Deeprails::Models::MonitorSubmitEventParams
|