deeprails 0.17.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +71 -0
- data/README.md +3 -3
- data/lib/deeprails/internal/stream.rb +29 -0
- data/lib/deeprails/internal/transport/base_client.rb +3 -1
- data/lib/deeprails/internal/transport/pooled_net_requester.rb +12 -10
- data/lib/deeprails/internal/type/base_stream.rb +83 -0
- data/lib/deeprails/internal/util.rb +7 -2
- data/lib/deeprails/models/defend_create_workflow_params.rb +14 -1
- data/lib/deeprails/models/defend_response.rb +136 -12
- data/lib/deeprails/models/defend_submit_and_stream_event_params.rb +83 -0
- data/lib/deeprails/models/defend_submit_and_stream_event_response.rb +7 -0
- data/lib/deeprails/models/defend_submit_event_params.rb +52 -16
- data/lib/deeprails/models/defend_update_response.rb +9 -1
- data/lib/deeprails/models/defend_update_workflow_params.rb +131 -5
- data/lib/deeprails/models/file_upload_params.rb +5 -5
- data/lib/deeprails/models/monitor_create_params.rb +14 -1
- data/lib/deeprails/models/monitor_detail_response.rb +56 -17
- data/lib/deeprails/models/monitor_event_detail_response.rb +6 -2
- data/lib/deeprails/models/monitor_submit_event_params.rb +52 -16
- data/lib/deeprails/models/monitor_update_params.rb +46 -5
- data/lib/deeprails/models/workflow_event_detail_response.rb +106 -34
- data/lib/deeprails/models/workflow_event_response.rb +9 -1
- data/lib/deeprails/models.rb +2 -0
- data/lib/deeprails/resources/defend.rb +70 -7
- data/lib/deeprails/resources/files.rb +2 -2
- data/lib/deeprails/resources/monitor.rb +14 -6
- data/lib/deeprails/version.rb +1 -1
- data/lib/deeprails.rb +5 -0
- data/manifest.yaml +1 -0
- data/rbi/deeprails/internal/stream.rbi +20 -0
- data/rbi/deeprails/internal/transport/base_client.rbi +18 -2
- data/rbi/deeprails/internal/type/base_stream.rbi +75 -0
- data/rbi/deeprails/models/defend_create_workflow_params.rbi +21 -0
- data/rbi/deeprails/models/defend_response.rbi +288 -16
- data/rbi/deeprails/models/defend_submit_and_stream_event_params.rbi +148 -0
- data/rbi/deeprails/models/defend_submit_and_stream_event_response.rbi +7 -0
- data/rbi/deeprails/models/defend_submit_event_params.rbi +108 -25
- data/rbi/deeprails/models/defend_update_response.rbi +14 -3
- data/rbi/deeprails/models/defend_update_workflow_params.rbi +298 -4
- data/rbi/deeprails/models/file_upload_params.rbi +9 -6
- data/rbi/deeprails/models/monitor_create_params.rbi +21 -0
- data/rbi/deeprails/models/monitor_detail_response.rbi +107 -26
- data/rbi/deeprails/models/monitor_event_detail_response.rbi +4 -2
- data/rbi/deeprails/models/monitor_submit_event_params.rbi +108 -25
- data/rbi/deeprails/models/monitor_update_params.rbi +107 -4
- data/rbi/deeprails/models/workflow_event_detail_response.rbi +209 -66
- data/rbi/deeprails/models/workflow_event_response.rbi +8 -0
- data/rbi/deeprails/models.rbi +3 -0
- data/rbi/deeprails/resources/defend.rbi +99 -9
- data/rbi/deeprails/resources/files.rbi +3 -3
- data/rbi/deeprails/resources/monitor.rbi +28 -8
- data/sig/deeprails/internal/stream.rbs +9 -0
- data/sig/deeprails/internal/type/base_stream.rbs +38 -0
- data/sig/deeprails/models/defend_create_workflow_params.rbs +7 -0
- data/sig/deeprails/models/defend_response.rbs +114 -8
- data/sig/deeprails/models/defend_submit_and_stream_event_params.rbs +74 -0
- data/sig/deeprails/models/defend_submit_and_stream_event_response.rbs +5 -0
- data/sig/deeprails/models/defend_submit_event_params.rbs +39 -10
- data/sig/deeprails/models/defend_update_response.rbs +10 -3
- data/sig/deeprails/models/defend_update_workflow_params.rbs +103 -1
- data/sig/deeprails/models/file_upload_params.rbs +4 -4
- data/sig/deeprails/models/monitor_create_params.rbs +7 -0
- data/sig/deeprails/models/monitor_detail_response.rbs +36 -9
- data/sig/deeprails/models/monitor_submit_event_params.rbs +39 -10
- data/sig/deeprails/models/monitor_update_params.rbs +45 -1
- data/sig/deeprails/models/workflow_event_detail_response.rbs +100 -27
- data/sig/deeprails/models/workflow_event_response.rbs +5 -0
- data/sig/deeprails/models.rbs +2 -0
- data/sig/deeprails/resources/defend.rbs +20 -0
- data/sig/deeprails/resources/files.rbs +1 -1
- data/sig/deeprails/resources/monitor.rbs +4 -0
- metadata +28 -2
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
module Deeprails
|
|
4
4
|
module Resources
|
|
5
5
|
class Defend
|
|
6
|
-
# Use this endpoint to create a new guardrail workflow
|
|
7
|
-
# thresholds
|
|
6
|
+
# Use this endpoint to create a new guardrail workflow by specifying guardrail
|
|
7
|
+
# thresholds, an improvement action, and optional extended capabilities.
|
|
8
8
|
sig do
|
|
9
9
|
params(
|
|
10
10
|
improvement_action:
|
|
@@ -17,6 +17,7 @@ module Deeprails
|
|
|
17
17
|
Symbol,
|
|
18
18
|
Deeprails::DefendCreateWorkflowParams::AutomaticHallucinationToleranceLevel::OrSymbol
|
|
19
19
|
],
|
|
20
|
+
context_awareness: T::Boolean,
|
|
20
21
|
custom_hallucination_threshold_values: T::Hash[Symbol, Float],
|
|
21
22
|
description: String,
|
|
22
23
|
file_search: T::Array[String],
|
|
@@ -45,6 +46,13 @@ module Deeprails
|
|
|
45
46
|
# `instruction_adherence`, `context_adherence`, `ground_truth_adherence`, or
|
|
46
47
|
# `comprehensive_safety`.
|
|
47
48
|
automatic_hallucination_tolerance_levels: nil,
|
|
49
|
+
# Context includes any structured information that directly relates to the model’s
|
|
50
|
+
# input and expected output—e.g., the recent turn-by-turn history between an AI
|
|
51
|
+
# tutor and a student, facts or state passed through an agentic workflow, or other
|
|
52
|
+
# domain-specific signals your system already knows and wants the model to
|
|
53
|
+
# condition on. This field determines whether to enable context awareness for this
|
|
54
|
+
# workflow's evaluations. Defaults to false.
|
|
55
|
+
context_awareness: nil,
|
|
48
56
|
# Mapping of guardrail metrics to floating point threshold values. Possible
|
|
49
57
|
# metrics are `correctness`, `completeness`, `instruction_adherence`,
|
|
50
58
|
# `context_adherence`, `ground_truth_adherence`, or `comprehensive_safety`.
|
|
@@ -98,6 +106,42 @@ module Deeprails
|
|
|
98
106
|
)
|
|
99
107
|
end
|
|
100
108
|
|
|
109
|
+
# Use this endpoint to create a new event for a guardrail workflow with real-time
|
|
110
|
+
# streaming feedback via Server-Sent Events (SSE).
|
|
111
|
+
sig do
|
|
112
|
+
params(
|
|
113
|
+
workflow_id: String,
|
|
114
|
+
model_input: T::Hash[Symbol, T.anything],
|
|
115
|
+
model_output: String,
|
|
116
|
+
model_used: String,
|
|
117
|
+
run_mode:
|
|
118
|
+
Deeprails::DefendSubmitAndStreamEventParams::RunMode::OrSymbol,
|
|
119
|
+
stream: T::Boolean,
|
|
120
|
+
nametag: String,
|
|
121
|
+
request_options: Deeprails::RequestOptions::OrHash
|
|
122
|
+
).returns(Deeprails::Internal::Stream[String])
|
|
123
|
+
end
|
|
124
|
+
def submit_and_stream_event_streaming(
|
|
125
|
+
# Path param: The ID of the workflow to create the event for.
|
|
126
|
+
workflow_id,
|
|
127
|
+
# Body param: The input provided to the model (e.g., prompt, messages).
|
|
128
|
+
model_input:,
|
|
129
|
+
# Body param: The output generated by the model to be evaluated.
|
|
130
|
+
model_output:,
|
|
131
|
+
# Body param: The model that generated the output (e.g., "gpt-4", "claude-3").
|
|
132
|
+
model_used:,
|
|
133
|
+
# Body param: The evaluation run mode. Streaming only supports fast, precision,
|
|
134
|
+
# and precision_codex.
|
|
135
|
+
run_mode:,
|
|
136
|
+
# Query param: Enable SSE streaming for real-time token feedback. Only supported
|
|
137
|
+
# for single-model run modes (fast, precision, precision_codex).
|
|
138
|
+
stream: nil,
|
|
139
|
+
# Body param: Optional tag to identify this event.
|
|
140
|
+
nametag: nil,
|
|
141
|
+
request_options: {}
|
|
142
|
+
)
|
|
143
|
+
end
|
|
144
|
+
|
|
101
145
|
# Use this endpoint to submit a model input and output pair to a workflow for
|
|
102
146
|
# evaluation
|
|
103
147
|
sig do
|
|
@@ -115,8 +159,8 @@ module Deeprails
|
|
|
115
159
|
# Workflow ID associated with this event.
|
|
116
160
|
workflow_id,
|
|
117
161
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
118
|
-
# contain
|
|
119
|
-
#
|
|
162
|
+
# contain a `user_prompt` field. For the ground_truth_adherence guardrail metric,
|
|
163
|
+
# `ground_truth` should be provided.
|
|
120
164
|
model_input:,
|
|
121
165
|
# Output generated by the LLM to be evaluated.
|
|
122
166
|
model_output:,
|
|
@@ -124,8 +168,8 @@ module Deeprails
|
|
|
124
168
|
model_used:,
|
|
125
169
|
# Run mode for the workflow event. The run mode allows the user to optimize for
|
|
126
170
|
# speed, accuracy, and cost by determining which models are used to evaluate the
|
|
127
|
-
# event. Available run modes include `
|
|
128
|
-
# `economy`. Defaults to `smart`.
|
|
171
|
+
# event. Available run modes include `precision_plus_codex`, `precision_plus`,
|
|
172
|
+
# `precision`, `smart`, and `economy`. Defaults to `smart`.
|
|
129
173
|
run_mode:,
|
|
130
174
|
# An optional, user-defined tag for the event.
|
|
131
175
|
nametag: nil,
|
|
@@ -133,22 +177,68 @@ module Deeprails
|
|
|
133
177
|
)
|
|
134
178
|
end
|
|
135
179
|
|
|
136
|
-
# Use this endpoint to update an existing defend workflow
|
|
180
|
+
# Use this endpoint to update an existing defend workflow if its details change.
|
|
137
181
|
sig do
|
|
138
182
|
params(
|
|
139
183
|
workflow_id: String,
|
|
184
|
+
automatic_hallucination_tolerance_levels:
|
|
185
|
+
T::Hash[
|
|
186
|
+
Symbol,
|
|
187
|
+
Deeprails::DefendUpdateWorkflowParams::AutomaticHallucinationToleranceLevel::OrSymbol
|
|
188
|
+
],
|
|
189
|
+
context_awareness: T::Boolean,
|
|
190
|
+
custom_hallucination_threshold_values: T::Hash[Symbol, Float],
|
|
140
191
|
description: String,
|
|
192
|
+
file_search: T::Array[String],
|
|
193
|
+
improvement_action:
|
|
194
|
+
Deeprails::DefendUpdateWorkflowParams::ImprovementAction::OrSymbol,
|
|
195
|
+
max_improvement_attempts: Integer,
|
|
141
196
|
name: String,
|
|
197
|
+
threshold_type:
|
|
198
|
+
Deeprails::DefendUpdateWorkflowParams::ThresholdType::OrSymbol,
|
|
199
|
+
web_search: T::Boolean,
|
|
142
200
|
request_options: Deeprails::RequestOptions::OrHash
|
|
143
201
|
).returns(Deeprails::DefendUpdateResponse)
|
|
144
202
|
end
|
|
145
203
|
def update_workflow(
|
|
146
204
|
# The ID of the workflow to edit.
|
|
147
205
|
workflow_id,
|
|
148
|
-
#
|
|
206
|
+
# New mapping of guardrail metrics to hallucination tolerance levels (either
|
|
207
|
+
# `low`, `medium`, or `high`) to be used when `threshold_type` is set to
|
|
208
|
+
# `automatic`. Possible metrics are `completeness`, `instruction_adherence`,
|
|
209
|
+
# `context_adherence`, `ground_truth_adherence`, or `comprehensive_safety`.
|
|
210
|
+
automatic_hallucination_tolerance_levels: nil,
|
|
211
|
+
# Whether to enable context awareness for this workflow's evaluations.
|
|
212
|
+
context_awareness: nil,
|
|
213
|
+
# New mapping of guardrail metrics to floating point threshold values to be used
|
|
214
|
+
# when `threshold_type` is set to `custom`. Possible metrics are `correctness`,
|
|
215
|
+
# `completeness`, `instruction_adherence`, `context_adherence`,
|
|
216
|
+
# `ground_truth_adherence`, or `comprehensive_safety`.
|
|
217
|
+
custom_hallucination_threshold_values: nil,
|
|
218
|
+
# New description for the workflow.
|
|
149
219
|
description: nil,
|
|
150
|
-
#
|
|
220
|
+
# An array of file IDs to search in the workflow's evaluations. Files must be
|
|
221
|
+
# uploaded via the DeepRails API first.
|
|
222
|
+
file_search: nil,
|
|
223
|
+
# The new action used to improve outputs that fail one or more guardrail metrics
|
|
224
|
+
# for the workflow events. May be `regen`, `fixit`, or `do_nothing`. ReGen runs
|
|
225
|
+
# the user's input prompt with minor induced variance. FixIt attempts to directly
|
|
226
|
+
# address the shortcomings of the output using the guardrail failure rationale. Do
|
|
227
|
+
# Nothing does not attempt any improvement.
|
|
228
|
+
improvement_action: nil,
|
|
229
|
+
# Max. number of improvement action attempts until a given event passes the
|
|
230
|
+
# guardrails. Defaults to 10.
|
|
231
|
+
max_improvement_attempts: nil,
|
|
232
|
+
# New name for the workflow.
|
|
151
233
|
name: nil,
|
|
234
|
+
# New type of thresholds to use for the workflow, either `automatic` or `custom`.
|
|
235
|
+
# Automatic thresholds are assigned internally after the user specifies a
|
|
236
|
+
# qualitative tolerance for the metrics, whereas custom metrics allow the user to
|
|
237
|
+
# set the threshold for each metric as a floating point number between 0.0 and
|
|
238
|
+
# 1.0.
|
|
239
|
+
threshold_type: nil,
|
|
240
|
+
# Whether to enable web search for this workflow's evaluations.
|
|
241
|
+
web_search: nil,
|
|
152
242
|
request_options: {}
|
|
153
243
|
)
|
|
154
244
|
end
|
|
@@ -6,13 +6,13 @@ module Deeprails
|
|
|
6
6
|
# Use this endpoint to upload a file to the DeepRails API
|
|
7
7
|
sig do
|
|
8
8
|
params(
|
|
9
|
-
|
|
9
|
+
files: T::Array[String],
|
|
10
10
|
request_options: Deeprails::RequestOptions::OrHash
|
|
11
11
|
).returns(Deeprails::FileResponse)
|
|
12
12
|
end
|
|
13
13
|
def upload(
|
|
14
|
-
# The contents of the
|
|
15
|
-
|
|
14
|
+
# The contents of the files to upload.
|
|
15
|
+
files:,
|
|
16
16
|
request_options: {}
|
|
17
17
|
)
|
|
18
18
|
end
|
|
@@ -10,6 +10,7 @@ module Deeprails
|
|
|
10
10
|
guardrail_metrics:
|
|
11
11
|
T::Array[Deeprails::MonitorCreateParams::GuardrailMetric::OrSymbol],
|
|
12
12
|
name: String,
|
|
13
|
+
context_awareness: T::Boolean,
|
|
13
14
|
description: String,
|
|
14
15
|
file_search: T::Array[String],
|
|
15
16
|
web_search: T::Boolean,
|
|
@@ -24,6 +25,13 @@ module Deeprails
|
|
|
24
25
|
guardrail_metrics:,
|
|
25
26
|
# Name of the new monitor.
|
|
26
27
|
name:,
|
|
28
|
+
# Context includes any structured information that directly relates to the model’s
|
|
29
|
+
# input and expected output—e.g., the recent turn-by-turn history between an AI
|
|
30
|
+
# tutor and a student, facts or state passed through an agentic workflow, or other
|
|
31
|
+
# domain-specific signals your system already knows and wants the model to
|
|
32
|
+
# condition on. This field determines whether to enable context awareness for this
|
|
33
|
+
# monitor's evaluations. Defaults to false.
|
|
34
|
+
context_awareness: nil,
|
|
27
35
|
# Description of the new monitor.
|
|
28
36
|
description: nil,
|
|
29
37
|
# An array of file IDs to search in the monitor's evaluations. Files must be
|
|
@@ -54,27 +62,39 @@ module Deeprails
|
|
|
54
62
|
)
|
|
55
63
|
end
|
|
56
64
|
|
|
57
|
-
# Use this endpoint to update the name,
|
|
58
|
-
# monitor
|
|
65
|
+
# Use this endpoint to update the name, status, and/or other details of an
|
|
66
|
+
# existing monitor.
|
|
59
67
|
sig do
|
|
60
68
|
params(
|
|
61
69
|
monitor_id: String,
|
|
62
70
|
description: String,
|
|
71
|
+
file_search: T::Array[String],
|
|
72
|
+
guardrail_metrics:
|
|
73
|
+
T::Array[Deeprails::MonitorUpdateParams::GuardrailMetric::OrSymbol],
|
|
63
74
|
name: String,
|
|
64
75
|
status: Deeprails::MonitorUpdateParams::Status::OrSymbol,
|
|
76
|
+
web_search: T::Boolean,
|
|
65
77
|
request_options: Deeprails::RequestOptions::OrHash
|
|
66
78
|
).returns(Deeprails::MonitorUpdateResponse)
|
|
67
79
|
end
|
|
68
80
|
def update(
|
|
69
81
|
# The ID of the monitor to edit.
|
|
70
82
|
monitor_id,
|
|
71
|
-
#
|
|
83
|
+
# New description of the monitor.
|
|
72
84
|
description: nil,
|
|
73
|
-
#
|
|
85
|
+
# An array of file IDs to search in the monitor's evaluations. Files must be
|
|
86
|
+
# uploaded via the DeepRails API first.
|
|
87
|
+
file_search: nil,
|
|
88
|
+
# An array of the new guardrail metrics that model input and output pairs will be
|
|
89
|
+
# evaluated on.
|
|
90
|
+
guardrail_metrics: nil,
|
|
91
|
+
# New name of the monitor.
|
|
74
92
|
name: nil,
|
|
75
93
|
# Status of the monitor. Can be `active` or `inactive`. Inactive monitors no
|
|
76
94
|
# longer record and evaluate events.
|
|
77
95
|
status: nil,
|
|
96
|
+
# Whether to enable web search for this monitor's evaluations.
|
|
97
|
+
web_search: nil,
|
|
78
98
|
request_options: {}
|
|
79
99
|
)
|
|
80
100
|
end
|
|
@@ -112,8 +132,8 @@ module Deeprails
|
|
|
112
132
|
# The ID of the monitor associated with this event.
|
|
113
133
|
monitor_id,
|
|
114
134
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
115
|
-
# contain
|
|
116
|
-
#
|
|
135
|
+
# contain a `user_prompt` field. For ground_truth_adherence guardrail metric,
|
|
136
|
+
# `ground_truth` should be provided.
|
|
117
137
|
model_input:,
|
|
118
138
|
# Output generated by the LLM to be evaluated.
|
|
119
139
|
model_output:,
|
|
@@ -121,8 +141,8 @@ module Deeprails
|
|
|
121
141
|
nametag: nil,
|
|
122
142
|
# Run mode for the monitor event. The run mode allows the user to optimize for
|
|
123
143
|
# speed, accuracy, and cost by determining which models are used to evaluate the
|
|
124
|
-
# event. Available run modes include `
|
|
125
|
-
# `economy`. Defaults to `smart`.
|
|
144
|
+
# event. Available run modes include `precision_plus_codex`, `precision_plus`,
|
|
145
|
+
# `precision`, `smart`, and `economy`. Defaults to `smart`.
|
|
126
146
|
run_mode: nil,
|
|
127
147
|
request_options: {}
|
|
128
148
|
)
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
module Deeprails
|
|
2
|
+
module Internal
|
|
3
|
+
module Type
|
|
4
|
+
module BaseStream[Message, Elem]
|
|
5
|
+
include Enumerable[Elem]
|
|
6
|
+
|
|
7
|
+
attr_reader status: Integer
|
|
8
|
+
|
|
9
|
+
attr_reader headers: ::Hash[String, String]
|
|
10
|
+
|
|
11
|
+
def close: -> void
|
|
12
|
+
|
|
13
|
+
private def iterator: -> Enumerable[Elem]
|
|
14
|
+
|
|
15
|
+
def each: { (Elem arg0) -> void } -> void
|
|
16
|
+
|
|
17
|
+
def to_enum: -> Enumerator[Elem]
|
|
18
|
+
|
|
19
|
+
alias enum_for to_enum
|
|
20
|
+
|
|
21
|
+
def initialize: (
|
|
22
|
+
model: Class | Deeprails::Internal::Type::Converter,
|
|
23
|
+
url: URI::Generic,
|
|
24
|
+
status: Integer,
|
|
25
|
+
headers: ::Hash[String, String],
|
|
26
|
+
response: top,
|
|
27
|
+
unwrap: Symbol
|
|
28
|
+
| Integer
|
|
29
|
+
| ::Array[Symbol | Integer]
|
|
30
|
+
| ^(top arg0) -> top,
|
|
31
|
+
stream: Enumerable[Message]
|
|
32
|
+
) -> void
|
|
33
|
+
|
|
34
|
+
def inspect: -> String
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
@@ -6,6 +6,7 @@ module Deeprails
|
|
|
6
6
|
name: String,
|
|
7
7
|
threshold_type: Deeprails::Models::DefendCreateWorkflowParams::threshold_type,
|
|
8
8
|
automatic_hallucination_tolerance_levels: ::Hash[Symbol, Deeprails::Models::DefendCreateWorkflowParams::automatic_hallucination_tolerance_level],
|
|
9
|
+
context_awareness: bool,
|
|
9
10
|
custom_hallucination_threshold_values: ::Hash[Symbol, Float],
|
|
10
11
|
description: String,
|
|
11
12
|
file_search: ::Array[String],
|
|
@@ -30,6 +31,10 @@ module Deeprails
|
|
|
30
31
|
::Hash[Symbol, Deeprails::Models::DefendCreateWorkflowParams::automatic_hallucination_tolerance_level]
|
|
31
32
|
) -> ::Hash[Symbol, Deeprails::Models::DefendCreateWorkflowParams::automatic_hallucination_tolerance_level]
|
|
32
33
|
|
|
34
|
+
attr_reader context_awareness: bool?
|
|
35
|
+
|
|
36
|
+
def context_awareness=: (bool) -> bool
|
|
37
|
+
|
|
33
38
|
attr_reader custom_hallucination_threshold_values: ::Hash[Symbol, Float]?
|
|
34
39
|
|
|
35
40
|
def custom_hallucination_threshold_values=: (
|
|
@@ -57,6 +62,7 @@ module Deeprails
|
|
|
57
62
|
name: String,
|
|
58
63
|
threshold_type: Deeprails::Models::DefendCreateWorkflowParams::threshold_type,
|
|
59
64
|
?automatic_hallucination_tolerance_levels: ::Hash[Symbol, Deeprails::Models::DefendCreateWorkflowParams::automatic_hallucination_tolerance_level],
|
|
65
|
+
?context_awareness: bool,
|
|
60
66
|
?custom_hallucination_threshold_values: ::Hash[Symbol, Float],
|
|
61
67
|
?description: String,
|
|
62
68
|
?file_search: ::Array[String],
|
|
@@ -70,6 +76,7 @@ module Deeprails
|
|
|
70
76
|
name: String,
|
|
71
77
|
threshold_type: Deeprails::Models::DefendCreateWorkflowParams::threshold_type,
|
|
72
78
|
automatic_hallucination_tolerance_levels: ::Hash[Symbol, Deeprails::Models::DefendCreateWorkflowParams::automatic_hallucination_tolerance_level],
|
|
79
|
+
context_awareness: bool,
|
|
73
80
|
custom_hallucination_threshold_values: ::Hash[Symbol, Float],
|
|
74
81
|
description: String,
|
|
75
82
|
file_search: ::Array[String],
|
|
@@ -115,13 +115,19 @@ module Deeprails
|
|
|
115
115
|
|
|
116
116
|
type event =
|
|
117
117
|
{
|
|
118
|
+
billing_request_id: String,
|
|
118
119
|
evaluations: ::Array[Deeprails::DefendResponse::Event::Evaluation],
|
|
119
120
|
event_id: String,
|
|
120
121
|
improved_model_output: String,
|
|
121
|
-
improvement_tool_status:
|
|
122
|
+
improvement_tool_status: Deeprails::Models::DefendResponse::Event::improvement_tool_status,
|
|
123
|
+
status: Deeprails::Models::DefendResponse::Event::status
|
|
122
124
|
}
|
|
123
125
|
|
|
124
126
|
class Event < Deeprails::Internal::Type::BaseModel
|
|
127
|
+
attr_reader billing_request_id: String?
|
|
128
|
+
|
|
129
|
+
def billing_request_id=: (String) -> String
|
|
130
|
+
|
|
125
131
|
attr_reader evaluations: ::Array[Deeprails::DefendResponse::Event::Evaluation]?
|
|
126
132
|
|
|
127
133
|
def evaluations=: (
|
|
@@ -136,26 +142,39 @@ module Deeprails
|
|
|
136
142
|
|
|
137
143
|
def improved_model_output=: (String) -> String
|
|
138
144
|
|
|
139
|
-
attr_reader improvement_tool_status:
|
|
145
|
+
attr_reader improvement_tool_status: Deeprails::Models::DefendResponse::Event::improvement_tool_status?
|
|
146
|
+
|
|
147
|
+
def improvement_tool_status=: (
|
|
148
|
+
Deeprails::Models::DefendResponse::Event::improvement_tool_status
|
|
149
|
+
) -> Deeprails::Models::DefendResponse::Event::improvement_tool_status
|
|
150
|
+
|
|
151
|
+
attr_reader status: Deeprails::Models::DefendResponse::Event::status?
|
|
140
152
|
|
|
141
|
-
def
|
|
153
|
+
def status=: (
|
|
154
|
+
Deeprails::Models::DefendResponse::Event::status
|
|
155
|
+
) -> Deeprails::Models::DefendResponse::Event::status
|
|
142
156
|
|
|
143
157
|
def initialize: (
|
|
158
|
+
?billing_request_id: String,
|
|
144
159
|
?evaluations: ::Array[Deeprails::DefendResponse::Event::Evaluation],
|
|
145
160
|
?event_id: String,
|
|
146
161
|
?improved_model_output: String,
|
|
147
|
-
?improvement_tool_status:
|
|
162
|
+
?improvement_tool_status: Deeprails::Models::DefendResponse::Event::improvement_tool_status,
|
|
163
|
+
?status: Deeprails::Models::DefendResponse::Event::status
|
|
148
164
|
) -> void
|
|
149
165
|
|
|
150
166
|
def to_hash: -> {
|
|
167
|
+
billing_request_id: String,
|
|
151
168
|
evaluations: ::Array[Deeprails::DefendResponse::Event::Evaluation],
|
|
152
169
|
event_id: String,
|
|
153
170
|
improved_model_output: String,
|
|
154
|
-
improvement_tool_status:
|
|
171
|
+
improvement_tool_status: Deeprails::Models::DefendResponse::Event::improvement_tool_status,
|
|
172
|
+
status: Deeprails::Models::DefendResponse::Event::status
|
|
155
173
|
}
|
|
156
174
|
|
|
157
175
|
type evaluation =
|
|
158
176
|
{
|
|
177
|
+
analysis_of_failures: String,
|
|
159
178
|
attempt: String,
|
|
160
179
|
created_at: Time,
|
|
161
180
|
error_message: String,
|
|
@@ -163,6 +182,8 @@ module Deeprails
|
|
|
163
182
|
evaluation_status: String,
|
|
164
183
|
evaluation_total_cost: Float,
|
|
165
184
|
guardrail_metrics: ::Array[String],
|
|
185
|
+
improvement_tool_status: Deeprails::Models::DefendResponse::Event::Evaluation::improvement_tool_status,
|
|
186
|
+
key_improvements: ::Array[String],
|
|
166
187
|
model_input: ::Hash[Symbol, top],
|
|
167
188
|
model_output: String,
|
|
168
189
|
modified_at: Time,
|
|
@@ -172,6 +193,10 @@ module Deeprails
|
|
|
172
193
|
}
|
|
173
194
|
|
|
174
195
|
class Evaluation < Deeprails::Internal::Type::BaseModel
|
|
196
|
+
attr_reader analysis_of_failures: String?
|
|
197
|
+
|
|
198
|
+
def analysis_of_failures=: (String) -> String
|
|
199
|
+
|
|
175
200
|
attr_reader attempt: String?
|
|
176
201
|
|
|
177
202
|
def attempt=: (String) -> String
|
|
@@ -200,6 +225,16 @@ module Deeprails
|
|
|
200
225
|
|
|
201
226
|
def guardrail_metrics=: (::Array[String]) -> ::Array[String]
|
|
202
227
|
|
|
228
|
+
attr_reader improvement_tool_status: Deeprails::Models::DefendResponse::Event::Evaluation::improvement_tool_status?
|
|
229
|
+
|
|
230
|
+
def improvement_tool_status=: (
|
|
231
|
+
Deeprails::Models::DefendResponse::Event::Evaluation::improvement_tool_status
|
|
232
|
+
) -> Deeprails::Models::DefendResponse::Event::Evaluation::improvement_tool_status
|
|
233
|
+
|
|
234
|
+
attr_reader key_improvements: ::Array[String]?
|
|
235
|
+
|
|
236
|
+
def key_improvements=: (::Array[String]) -> ::Array[String]
|
|
237
|
+
|
|
203
238
|
attr_reader model_input: ::Hash[Symbol, top]?
|
|
204
239
|
|
|
205
240
|
def model_input=: (::Hash[Symbol, top]) -> ::Hash[Symbol, top]
|
|
@@ -225,6 +260,7 @@ module Deeprails
|
|
|
225
260
|
def run_mode=: (String) -> String
|
|
226
261
|
|
|
227
262
|
def initialize: (
|
|
263
|
+
?analysis_of_failures: String,
|
|
228
264
|
?attempt: String,
|
|
229
265
|
?created_at: Time,
|
|
230
266
|
?error_message: String,
|
|
@@ -232,6 +268,8 @@ module Deeprails
|
|
|
232
268
|
?evaluation_status: String,
|
|
233
269
|
?evaluation_total_cost: Float,
|
|
234
270
|
?guardrail_metrics: ::Array[String],
|
|
271
|
+
?improvement_tool_status: Deeprails::Models::DefendResponse::Event::Evaluation::improvement_tool_status,
|
|
272
|
+
?key_improvements: ::Array[String],
|
|
235
273
|
?model_input: ::Hash[Symbol, top],
|
|
236
274
|
?model_output: String,
|
|
237
275
|
?modified_at: Time,
|
|
@@ -241,6 +279,7 @@ module Deeprails
|
|
|
241
279
|
) -> void
|
|
242
280
|
|
|
243
281
|
def to_hash: -> {
|
|
282
|
+
analysis_of_failures: String,
|
|
244
283
|
attempt: String,
|
|
245
284
|
created_at: Time,
|
|
246
285
|
error_message: String,
|
|
@@ -248,6 +287,8 @@ module Deeprails
|
|
|
248
287
|
evaluation_status: String,
|
|
249
288
|
evaluation_total_cost: Float,
|
|
250
289
|
guardrail_metrics: ::Array[String],
|
|
290
|
+
improvement_tool_status: Deeprails::Models::DefendResponse::Event::Evaluation::improvement_tool_status,
|
|
291
|
+
key_improvements: ::Array[String],
|
|
251
292
|
model_input: ::Hash[Symbol, top],
|
|
252
293
|
model_output: String,
|
|
253
294
|
modified_at: Time,
|
|
@@ -255,10 +296,63 @@ module Deeprails
|
|
|
255
296
|
progress: Integer,
|
|
256
297
|
run_mode: String
|
|
257
298
|
}
|
|
299
|
+
|
|
300
|
+
type improvement_tool_status =
|
|
301
|
+
:improved
|
|
302
|
+
| :improvement_failed
|
|
303
|
+
| :no_improvement_required
|
|
304
|
+
| :improvement_required
|
|
305
|
+
|
|
306
|
+
module ImprovementToolStatus
|
|
307
|
+
extend Deeprails::Internal::Type::Enum
|
|
308
|
+
|
|
309
|
+
IMPROVED: :improved
|
|
310
|
+
IMPROVEMENT_FAILED: :improvement_failed
|
|
311
|
+
NO_IMPROVEMENT_REQUIRED: :no_improvement_required
|
|
312
|
+
IMPROVEMENT_REQUIRED: :improvement_required
|
|
313
|
+
|
|
314
|
+
def self?.values: -> ::Array[Deeprails::Models::DefendResponse::Event::Evaluation::improvement_tool_status]
|
|
315
|
+
end
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
type improvement_tool_status =
|
|
319
|
+
:improved
|
|
320
|
+
| :improvement_failed
|
|
321
|
+
| :no_improvement_required
|
|
322
|
+
| :improvement_required
|
|
323
|
+
|
|
324
|
+
module ImprovementToolStatus
|
|
325
|
+
extend Deeprails::Internal::Type::Enum
|
|
326
|
+
|
|
327
|
+
IMPROVED: :improved
|
|
328
|
+
IMPROVEMENT_FAILED: :improvement_failed
|
|
329
|
+
NO_IMPROVEMENT_REQUIRED: :no_improvement_required
|
|
330
|
+
IMPROVEMENT_REQUIRED: :improvement_required
|
|
331
|
+
|
|
332
|
+
def self?.values: -> ::Array[Deeprails::Models::DefendResponse::Event::improvement_tool_status]
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
type status = :completed | :failed | :in_progress
|
|
336
|
+
|
|
337
|
+
module Status
|
|
338
|
+
extend Deeprails::Internal::Type::Enum
|
|
339
|
+
|
|
340
|
+
COMPLETED: :completed
|
|
341
|
+
FAILED: :failed
|
|
342
|
+
IN_PROGRESS: :in_progress
|
|
343
|
+
|
|
344
|
+
def self?.values: -> ::Array[Deeprails::Models::DefendResponse::Event::status]
|
|
258
345
|
end
|
|
259
346
|
end
|
|
260
347
|
|
|
261
|
-
type file =
|
|
348
|
+
type file =
|
|
349
|
+
{
|
|
350
|
+
file_id: String,
|
|
351
|
+
file_name: String,
|
|
352
|
+
file_size: Integer,
|
|
353
|
+
presigned_url: String,
|
|
354
|
+
presigned_url_expires_at: Time
|
|
355
|
+
}
|
|
262
356
|
|
|
263
357
|
class File < Deeprails::Internal::Type::BaseModel
|
|
264
358
|
attr_reader file_id: String?
|
|
@@ -273,16 +367,28 @@ module Deeprails
|
|
|
273
367
|
|
|
274
368
|
def file_size=: (Integer) -> Integer
|
|
275
369
|
|
|
370
|
+
attr_reader presigned_url: String?
|
|
371
|
+
|
|
372
|
+
def presigned_url=: (String) -> String
|
|
373
|
+
|
|
374
|
+
attr_reader presigned_url_expires_at: Time?
|
|
375
|
+
|
|
376
|
+
def presigned_url_expires_at=: (Time) -> Time
|
|
377
|
+
|
|
276
378
|
def initialize: (
|
|
277
379
|
?file_id: String,
|
|
278
380
|
?file_name: String,
|
|
279
|
-
?file_size: Integer
|
|
381
|
+
?file_size: Integer,
|
|
382
|
+
?presigned_url: String,
|
|
383
|
+
?presigned_url_expires_at: Time
|
|
280
384
|
) -> void
|
|
281
385
|
|
|
282
386
|
def to_hash: -> {
|
|
283
387
|
file_id: String,
|
|
284
388
|
file_name: String,
|
|
285
|
-
file_size: Integer
|
|
389
|
+
file_size: Integer,
|
|
390
|
+
presigned_url: String,
|
|
391
|
+
presigned_url_expires_at: Time
|
|
286
392
|
}
|
|
287
393
|
end
|
|
288
394
|
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
module Deeprails
|
|
2
|
+
module Models
|
|
3
|
+
type defend_submit_and_stream_event_params =
|
|
4
|
+
{
|
|
5
|
+
model_input: ::Hash[Symbol, top],
|
|
6
|
+
model_output: String,
|
|
7
|
+
model_used: String,
|
|
8
|
+
run_mode: Deeprails::Models::DefendSubmitAndStreamEventParams::run_mode,
|
|
9
|
+
stream: bool,
|
|
10
|
+
nametag: String
|
|
11
|
+
}
|
|
12
|
+
& Deeprails::Internal::Type::request_parameters
|
|
13
|
+
|
|
14
|
+
class DefendSubmitAndStreamEventParams < Deeprails::Internal::Type::BaseModel
|
|
15
|
+
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
16
|
+
include Deeprails::Internal::Type::RequestParameters
|
|
17
|
+
|
|
18
|
+
attr_accessor model_input: ::Hash[Symbol, top]
|
|
19
|
+
|
|
20
|
+
attr_accessor model_output: String
|
|
21
|
+
|
|
22
|
+
attr_accessor model_used: String
|
|
23
|
+
|
|
24
|
+
attr_accessor run_mode: Deeprails::Models::DefendSubmitAndStreamEventParams::run_mode
|
|
25
|
+
|
|
26
|
+
attr_reader stream: bool?
|
|
27
|
+
|
|
28
|
+
def stream=: (bool) -> bool
|
|
29
|
+
|
|
30
|
+
attr_reader nametag: String?
|
|
31
|
+
|
|
32
|
+
def nametag=: (String) -> String
|
|
33
|
+
|
|
34
|
+
def initialize: (
|
|
35
|
+
model_input: ::Hash[Symbol, top],
|
|
36
|
+
model_output: String,
|
|
37
|
+
model_used: String,
|
|
38
|
+
run_mode: Deeprails::Models::DefendSubmitAndStreamEventParams::run_mode,
|
|
39
|
+
?stream: bool,
|
|
40
|
+
?nametag: String,
|
|
41
|
+
?request_options: Deeprails::request_opts
|
|
42
|
+
) -> void
|
|
43
|
+
|
|
44
|
+
def to_hash: -> {
|
|
45
|
+
model_input: ::Hash[Symbol, top],
|
|
46
|
+
model_output: String,
|
|
47
|
+
model_used: String,
|
|
48
|
+
run_mode: Deeprails::Models::DefendSubmitAndStreamEventParams::run_mode,
|
|
49
|
+
stream: bool,
|
|
50
|
+
nametag: String,
|
|
51
|
+
request_options: Deeprails::RequestOptions
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
type run_mode =
|
|
55
|
+
:fast
|
|
56
|
+
| :precision
|
|
57
|
+
| :precision_codex
|
|
58
|
+
| :precision_max
|
|
59
|
+
| :precision_max_codex
|
|
60
|
+
|
|
61
|
+
module RunMode
|
|
62
|
+
extend Deeprails::Internal::Type::Enum
|
|
63
|
+
|
|
64
|
+
FAST: :fast
|
|
65
|
+
PRECISION: :precision
|
|
66
|
+
PRECISION_CODEX: :precision_codex
|
|
67
|
+
PRECISION_MAX: :precision_max
|
|
68
|
+
PRECISION_MAX_CODEX: :precision_max_codex
|
|
69
|
+
|
|
70
|
+
def self?.values: -> ::Array[Deeprails::Models::DefendSubmitAndStreamEventParams::run_mode]
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|