deeprails 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/README.md +19 -19
- data/lib/deeprails/models/defend_create_workflow_params.rb +37 -39
- data/lib/deeprails/models/defend_response.rb +20 -19
- data/lib/deeprails/models/defend_submit_event_params.rb +4 -4
- data/lib/deeprails/models/evaluate_create_params.rb +4 -4
- data/lib/deeprails/models/evaluation.rb +4 -4
- data/lib/deeprails/models/monitor_submit_event_params.rb +4 -4
- data/lib/deeprails/resources/defend.rb +6 -6
- data/lib/deeprails/version.rb +1 -1
- data/rbi/deeprails/models/defend_create_workflow_params.rbi +83 -67
- data/rbi/deeprails/models/defend_response.rbi +41 -30
- data/rbi/deeprails/models/defend_submit_event_params.rbi +6 -6
- data/rbi/deeprails/models/evaluate_create_params.rbi +6 -6
- data/rbi/deeprails/models/evaluation.rbi +6 -6
- data/rbi/deeprails/models/monitor_submit_event_params.rbi +6 -6
- data/rbi/deeprails/resources/defend.rbi +24 -23
- data/rbi/deeprails/resources/evaluate.rbi +2 -2
- data/rbi/deeprails/resources/monitor.rbi +2 -2
- data/sig/deeprails/models/defend_create_workflow_params.rbs +31 -26
- data/sig/deeprails/models/defend_response.rbs +18 -13
- data/sig/deeprails/resources/defend.rbs +4 -4
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e7d722f2acb3079695e92cd5455050c9d6b09fb46687b4300fba8eee40f228bf
|
|
4
|
+
data.tar.gz: 5783eb1fdf16fd00387b893187d51d2bf8a157d3f5d51e2318dc525761045da8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 807da14f2454695d4eb036637f6a645985b12aa3e821171aaeec65cd5964ed86086afd822391ea85b8358631c3cd3c73b1b4c262e0e9429a171711e5870552a6
|
|
7
|
+
data.tar.gz: 659273be50dd4dd1f8b43fe5a5d35640359a6b656d51298275c542f3996170bad7ebc7f0dc56b7e914264513eef1595b80fbee4c35e32f2036ae092aea28b26c
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.7.0 (2025-10-22)
|
|
4
|
+
|
|
5
|
+
Full Changelog: [v0.6.0...v0.7.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.6.0...v0.7.0)
|
|
6
|
+
|
|
7
|
+
### Features
|
|
8
|
+
|
|
9
|
+
* **api:** manual updates ([3f3207a](https://github.com/deeprails/deeprails-ruby-sdk/commit/3f3207ad9b0e71ed6911c94cf6850a4d35a8d295))
|
|
10
|
+
|
|
11
|
+
## 0.6.0 (2025-10-22)
|
|
12
|
+
|
|
13
|
+
Full Changelog: [v0.5.0...v0.6.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.5.0...v0.6.0)
|
|
14
|
+
|
|
15
|
+
### Features
|
|
16
|
+
|
|
17
|
+
* **api:** update defend naming and data structures ([e345a29](https://github.com/deeprails/deeprails-ruby-sdk/commit/e345a293b74cc3bfedcfcf8b5d9c9a834f82a69b))
|
|
18
|
+
|
|
3
19
|
## 0.5.0 (2025-10-20)
|
|
4
20
|
|
|
5
21
|
Full Changelog: [v0.4.0...v0.5.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.4.0...v0.5.0)
|
data/README.md
CHANGED
|
@@ -15,7 +15,7 @@ To use this gem, install via Bundler by adding the following to your application
|
|
|
15
15
|
<!-- x-release-please-start-version -->
|
|
16
16
|
|
|
17
17
|
```ruby
|
|
18
|
-
gem "deeprails", "~> 0.
|
|
18
|
+
gem "deeprails", "~> 0.7.0"
|
|
19
19
|
```
|
|
20
20
|
|
|
21
21
|
<!-- x-release-please-end -->
|
|
@@ -32,9 +32,9 @@ deeprails = Deeprails::Client.new(
|
|
|
32
32
|
|
|
33
33
|
defend_response = deeprails.defend.create_workflow(
|
|
34
34
|
improvement_action: "fixit",
|
|
35
|
-
metrics: {completeness: 0.7, instruction_adherence: 0.75},
|
|
36
35
|
name: "Push Alert Workflow",
|
|
37
|
-
type: "custom"
|
|
36
|
+
type: "custom",
|
|
37
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75}
|
|
38
38
|
)
|
|
39
39
|
|
|
40
40
|
puts(defend_response.workflow_id)
|
|
@@ -48,9 +48,9 @@ When the library is unable to connect to the API, or if the API returns a non-su
|
|
|
48
48
|
begin
|
|
49
49
|
defend = deeprails.defend.create_workflow(
|
|
50
50
|
improvement_action: "fixit",
|
|
51
|
-
metrics: {completeness: 0.7, instruction_adherence: 0.75},
|
|
52
51
|
name: "Push Alert Workflow",
|
|
53
|
-
type: "custom"
|
|
52
|
+
type: "custom",
|
|
53
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75}
|
|
54
54
|
)
|
|
55
55
|
rescue Deeprails::Errors::APIConnectionError => e
|
|
56
56
|
puts("The server could not be reached")
|
|
@@ -96,9 +96,9 @@ deeprails = Deeprails::Client.new(
|
|
|
96
96
|
# Or, configure per-request:
|
|
97
97
|
deeprails.defend.create_workflow(
|
|
98
98
|
improvement_action: "fixit",
|
|
99
|
-
metrics: {completeness: 0.7, instruction_adherence: 0.75},
|
|
100
99
|
name: "Push Alert Workflow",
|
|
101
100
|
type: "custom",
|
|
101
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75},
|
|
102
102
|
request_options: {max_retries: 5}
|
|
103
103
|
)
|
|
104
104
|
```
|
|
@@ -116,9 +116,9 @@ deeprails = Deeprails::Client.new(
|
|
|
116
116
|
# Or, configure per-request:
|
|
117
117
|
deeprails.defend.create_workflow(
|
|
118
118
|
improvement_action: "fixit",
|
|
119
|
-
metrics: {completeness: 0.7, instruction_adherence: 0.75},
|
|
120
119
|
name: "Push Alert Workflow",
|
|
121
120
|
type: "custom",
|
|
121
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75},
|
|
122
122
|
request_options: {timeout: 5}
|
|
123
123
|
)
|
|
124
124
|
```
|
|
@@ -153,9 +153,9 @@ Note: the `extra_` parameters of the same name overrides the documented paramete
|
|
|
153
153
|
defend_response =
|
|
154
154
|
deeprails.defend.create_workflow(
|
|
155
155
|
improvement_action: "fixit",
|
|
156
|
-
metrics: {completeness: 0.7, instruction_adherence: 0.75},
|
|
157
156
|
name: "Push Alert Workflow",
|
|
158
157
|
type: "custom",
|
|
158
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75},
|
|
159
159
|
request_options: {
|
|
160
160
|
extra_query: {my_query_parameter: value},
|
|
161
161
|
extra_body: {my_body_parameter: value},
|
|
@@ -203,9 +203,9 @@ You can provide typesafe request parameters like so:
|
|
|
203
203
|
```ruby
|
|
204
204
|
deeprails.defend.create_workflow(
|
|
205
205
|
improvement_action: "fixit",
|
|
206
|
-
metrics: {completeness: 0.7, instruction_adherence: 0.75},
|
|
207
206
|
name: "Push Alert Workflow",
|
|
208
|
-
type: "custom"
|
|
207
|
+
type: "custom",
|
|
208
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75}
|
|
209
209
|
)
|
|
210
210
|
```
|
|
211
211
|
|
|
@@ -215,17 +215,17 @@ Or, equivalently:
|
|
|
215
215
|
# Hashes work, but are not typesafe:
|
|
216
216
|
deeprails.defend.create_workflow(
|
|
217
217
|
improvement_action: "fixit",
|
|
218
|
-
metrics: {completeness: 0.7, instruction_adherence: 0.75},
|
|
219
218
|
name: "Push Alert Workflow",
|
|
220
|
-
type: "custom"
|
|
219
|
+
type: "custom",
|
|
220
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75}
|
|
221
221
|
)
|
|
222
222
|
|
|
223
223
|
# You can also splat a full Params class:
|
|
224
224
|
params = Deeprails::DefendCreateWorkflowParams.new(
|
|
225
225
|
improvement_action: "fixit",
|
|
226
|
-
metrics: {completeness: 0.7, instruction_adherence: 0.75},
|
|
227
226
|
name: "Push Alert Workflow",
|
|
228
|
-
type: "custom"
|
|
227
|
+
type: "custom",
|
|
228
|
+
custom_hallucination_threshold_values: {completeness: 0.7, instruction_adherence: 0.75}
|
|
229
229
|
)
|
|
230
230
|
deeprails.defend.create_workflow(**params)
|
|
231
231
|
```
|
|
@@ -235,11 +235,11 @@ deeprails.defend.create_workflow(**params)
|
|
|
235
235
|
Since this library does not depend on `sorbet-runtime`, it cannot provide [`T::Enum`](https://sorbet.org/docs/tenum) instances. Instead, we provide "tagged symbols" instead, which is always a primitive at runtime:
|
|
236
236
|
|
|
237
237
|
```ruby
|
|
238
|
-
# :
|
|
239
|
-
puts(Deeprails::DefendCreateWorkflowParams::ImprovementAction::
|
|
238
|
+
# :regen
|
|
239
|
+
puts(Deeprails::DefendCreateWorkflowParams::ImprovementAction::REGEN)
|
|
240
240
|
|
|
241
241
|
# Revealed type: `T.all(Deeprails::DefendCreateWorkflowParams::ImprovementAction, Symbol)`
|
|
242
|
-
T.reveal_type(Deeprails::DefendCreateWorkflowParams::ImprovementAction::
|
|
242
|
+
T.reveal_type(Deeprails::DefendCreateWorkflowParams::ImprovementAction::REGEN)
|
|
243
243
|
```
|
|
244
244
|
|
|
245
245
|
Enum parameters have a "relaxed" type, so you can either pass in enum constants or their literal value:
|
|
@@ -247,13 +247,13 @@ Enum parameters have a "relaxed" type, so you can either pass in enum constants
|
|
|
247
247
|
```ruby
|
|
248
248
|
# Using the enum constants preserves the tagged type information:
|
|
249
249
|
deeprails.defend.create_workflow(
|
|
250
|
-
improvement_action: Deeprails::DefendCreateWorkflowParams::ImprovementAction::
|
|
250
|
+
improvement_action: Deeprails::DefendCreateWorkflowParams::ImprovementAction::REGEN,
|
|
251
251
|
# …
|
|
252
252
|
)
|
|
253
253
|
|
|
254
254
|
# Literal values are also permissible:
|
|
255
255
|
deeprails.defend.create_workflow(
|
|
256
|
-
improvement_action: :
|
|
256
|
+
improvement_action: :regen,
|
|
257
257
|
# …
|
|
258
258
|
)
|
|
259
259
|
```
|
|
@@ -9,25 +9,13 @@ module Deeprails
|
|
|
9
9
|
|
|
10
10
|
# @!attribute improvement_action
|
|
11
11
|
# The action used to improve outputs that fail one or guardrail metrics for the
|
|
12
|
-
# workflow events. May be `
|
|
13
|
-
#
|
|
14
|
-
#
|
|
15
|
-
#
|
|
12
|
+
# workflow events. May be `regen`, `fixit`, or `do_nothing`. ReGen runs the user's
|
|
13
|
+
# input prompt with minor induced variance. FixIt attempts to directly address the
|
|
14
|
+
# shortcomings of the output using the guardrail failure rationale. Do Nothing
|
|
15
|
+
# does not attempt any improvement.
|
|
16
16
|
#
|
|
17
|
-
# @return [Symbol, Deeprails::Models::DefendCreateWorkflowParams::ImprovementAction
|
|
18
|
-
required :improvement_action,
|
|
19
|
-
enum: -> { Deeprails::DefendCreateWorkflowParams::ImprovementAction },
|
|
20
|
-
nil?: true
|
|
21
|
-
|
|
22
|
-
# @!attribute metrics
|
|
23
|
-
# Mapping of guardrail metrics to floating point threshold values. If the workflow
|
|
24
|
-
# type is automatic, only the metric names are used (`automatic_tolerance`
|
|
25
|
-
# determines thresholds). Possible metrics are `correctness`, `completeness`,
|
|
26
|
-
# `instruction_adherence`, `context_adherence`, `ground_truth_adherence`, or
|
|
27
|
-
# `comprehensive_safety`.
|
|
28
|
-
#
|
|
29
|
-
# @return [Hash{Symbol=>Float}]
|
|
30
|
-
required :metrics, Deeprails::Internal::Type::HashOf[Float]
|
|
17
|
+
# @return [Symbol, Deeprails::Models::DefendCreateWorkflowParams::ImprovementAction]
|
|
18
|
+
required :improvement_action, enum: -> { Deeprails::DefendCreateWorkflowParams::ImprovementAction }
|
|
31
19
|
|
|
32
20
|
# @!attribute name
|
|
33
21
|
# Name of the workflow.
|
|
@@ -45,12 +33,23 @@ module Deeprails
|
|
|
45
33
|
# @return [Symbol, Deeprails::Models::DefendCreateWorkflowParams::Type]
|
|
46
34
|
required :type, enum: -> { Deeprails::DefendCreateWorkflowParams::Type }
|
|
47
35
|
|
|
48
|
-
# @!attribute
|
|
49
|
-
#
|
|
50
|
-
# `
|
|
36
|
+
# @!attribute automatic_hallucination_tolerance_levels
|
|
37
|
+
# Mapping of guardrail metrics to hallucination tolerance levels (either `low`,
|
|
38
|
+
# `medium`, or `high`). Possible metrics are `completeness`,
|
|
39
|
+
# `instruction_adherence`, `context_adherence`, `ground_truth_adherence`, or
|
|
40
|
+
# `comprehensive_safety`.
|
|
41
|
+
#
|
|
42
|
+
# @return [Hash{Symbol=>Symbol, Deeprails::Models::DefendCreateWorkflowParams::AutomaticHallucinationToleranceLevel}, nil]
|
|
43
|
+
optional :automatic_hallucination_tolerance_levels,
|
|
44
|
+
-> { Deeprails::Internal::Type::HashOf[enum: Deeprails::DefendCreateWorkflowParams::AutomaticHallucinationToleranceLevel] }
|
|
45
|
+
|
|
46
|
+
# @!attribute custom_hallucination_threshold_values
|
|
47
|
+
# Mapping of guardrail metrics to floating point threshold values. Possible
|
|
48
|
+
# metrics are `correctness`, `completeness`, `instruction_adherence`,
|
|
49
|
+
# `context_adherence`, `ground_truth_adherence`, or `comprehensive_safety`.
|
|
51
50
|
#
|
|
52
|
-
# @return [Symbol,
|
|
53
|
-
optional :
|
|
51
|
+
# @return [Hash{Symbol=>Float}, nil]
|
|
52
|
+
optional :custom_hallucination_threshold_values, Deeprails::Internal::Type::HashOf[Float]
|
|
54
53
|
|
|
55
54
|
# @!attribute description
|
|
56
55
|
# Description for the workflow.
|
|
@@ -58,43 +57,44 @@ module Deeprails
|
|
|
58
57
|
# @return [String, nil]
|
|
59
58
|
optional :description, String
|
|
60
59
|
|
|
61
|
-
# @!attribute
|
|
60
|
+
# @!attribute max_improvement_attempt
|
|
62
61
|
# Max. number of improvement action retries until a given event passes the
|
|
63
62
|
# guardrails. Defaults to 10.
|
|
64
63
|
#
|
|
65
64
|
# @return [Integer, nil]
|
|
66
|
-
optional :
|
|
65
|
+
optional :max_improvement_attempt, Integer
|
|
67
66
|
|
|
68
|
-
# @!method initialize(improvement_action:,
|
|
67
|
+
# @!method initialize(improvement_action:, name:, type:, automatic_hallucination_tolerance_levels: nil, custom_hallucination_threshold_values: nil, description: nil, max_improvement_attempt: nil, request_options: {})
|
|
69
68
|
# Some parameter documentations has been truncated, see
|
|
70
69
|
# {Deeprails::Models::DefendCreateWorkflowParams} for more details.
|
|
71
70
|
#
|
|
72
|
-
# @param improvement_action [Symbol, Deeprails::Models::DefendCreateWorkflowParams::ImprovementAction
|
|
73
|
-
#
|
|
74
|
-
# @param metrics [Hash{Symbol=>Float}] Mapping of guardrail metrics to floating point threshold values. If the workflo
|
|
71
|
+
# @param improvement_action [Symbol, Deeprails::Models::DefendCreateWorkflowParams::ImprovementAction] The action used to improve outputs that fail one or guardrail metrics for the wo
|
|
75
72
|
#
|
|
76
73
|
# @param name [String] Name of the workflow.
|
|
77
74
|
#
|
|
78
75
|
# @param type [Symbol, Deeprails::Models::DefendCreateWorkflowParams::Type] Type of thresholds to use for the workflow, either `automatic` or `custom`. Aut
|
|
79
76
|
#
|
|
80
|
-
# @param
|
|
77
|
+
# @param automatic_hallucination_tolerance_levels [Hash{Symbol=>Symbol, Deeprails::Models::DefendCreateWorkflowParams::AutomaticHallucinationToleranceLevel}] Mapping of guardrail metrics to hallucination tolerance levels
|
|
78
|
+
#
|
|
79
|
+
# @param custom_hallucination_threshold_values [Hash{Symbol=>Float}] Mapping of guardrail metrics to floating point threshold values. Possible metric
|
|
81
80
|
#
|
|
82
81
|
# @param description [String] Description for the workflow.
|
|
83
82
|
#
|
|
84
|
-
# @param
|
|
83
|
+
# @param max_improvement_attempt [Integer] Max. number of improvement action retries until a given event passes the guardra
|
|
85
84
|
#
|
|
86
85
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
87
86
|
|
|
88
87
|
# The action used to improve outputs that fail one or guardrail metrics for the
|
|
89
|
-
# workflow events. May be `
|
|
90
|
-
#
|
|
91
|
-
#
|
|
92
|
-
#
|
|
88
|
+
# workflow events. May be `regen`, `fixit`, or `do_nothing`. ReGen runs the user's
|
|
89
|
+
# input prompt with minor induced variance. FixIt attempts to directly address the
|
|
90
|
+
# shortcomings of the output using the guardrail failure rationale. Do Nothing
|
|
91
|
+
# does not attempt any improvement.
|
|
93
92
|
module ImprovementAction
|
|
94
93
|
extend Deeprails::Internal::Type::Enum
|
|
95
94
|
|
|
96
|
-
|
|
95
|
+
REGEN = :regen
|
|
97
96
|
FIXIT = :fixit
|
|
97
|
+
DO_NOTHING = :do_nothing
|
|
98
98
|
|
|
99
99
|
# @!method self.values
|
|
100
100
|
# @return [Array<Symbol>]
|
|
@@ -115,9 +115,7 @@ module Deeprails
|
|
|
115
115
|
# @return [Array<Symbol>]
|
|
116
116
|
end
|
|
117
117
|
|
|
118
|
-
|
|
119
|
-
# `high`. Ignored if `type` is `custom`.
|
|
120
|
-
module AutomaticTolerance
|
|
118
|
+
module AutomaticHallucinationToleranceLevel
|
|
121
119
|
extend Deeprails::Internal::Type::Enum
|
|
122
120
|
|
|
123
121
|
LOW = :low
|
|
@@ -30,20 +30,20 @@ module Deeprails
|
|
|
30
30
|
|
|
31
31
|
# @!attribute improvement_action
|
|
32
32
|
# The action used to improve outputs that fail one or more guardrail metrics for
|
|
33
|
-
# the workflow events. May be `
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
#
|
|
33
|
+
# the workflow events. May be `regen`, `fixit`, or `do_nothing`. ReGen runs the
|
|
34
|
+
# user's input prompt with minor induced variance. FixIt attempts to directly
|
|
35
|
+
# address the shortcomings of the output using the guardrail failure rationale. Do
|
|
36
|
+
# Nothing does not attempt any improvement.
|
|
37
37
|
#
|
|
38
38
|
# @return [Symbol, Deeprails::Models::DefendResponse::ImprovementAction, nil]
|
|
39
|
-
optional :improvement_action, enum: -> { Deeprails::DefendResponse::ImprovementAction }
|
|
39
|
+
optional :improvement_action, enum: -> { Deeprails::DefendResponse::ImprovementAction }
|
|
40
40
|
|
|
41
|
-
# @!attribute
|
|
41
|
+
# @!attribute max_improvement_attempt
|
|
42
42
|
# Max. number of improvement action retries until a given event passes the
|
|
43
43
|
# guardrails.
|
|
44
44
|
#
|
|
45
45
|
# @return [Integer, nil]
|
|
46
|
-
optional :
|
|
46
|
+
optional :max_improvement_attempt, Integer
|
|
47
47
|
|
|
48
48
|
# @!attribute modified_at
|
|
49
49
|
# The most recent time the workflow was modified in UTC.
|
|
@@ -52,7 +52,7 @@ module Deeprails
|
|
|
52
52
|
optional :modified_at, Time
|
|
53
53
|
|
|
54
54
|
# @!attribute status
|
|
55
|
-
# Status of the selected workflow. May be `
|
|
55
|
+
# Status of the selected workflow. May be `inactive` or `active`. Inactive
|
|
56
56
|
# workflows will not accept events.
|
|
57
57
|
#
|
|
58
58
|
# @return [Symbol, Deeprails::Models::DefendResponse::Status, nil]
|
|
@@ -64,7 +64,7 @@ module Deeprails
|
|
|
64
64
|
# @return [Float, nil]
|
|
65
65
|
optional :success_rate, Float
|
|
66
66
|
|
|
67
|
-
# @!method initialize(name:, workflow_id:, created_at: nil, description: nil, improvement_action: nil,
|
|
67
|
+
# @!method initialize(name:, workflow_id:, created_at: nil, description: nil, improvement_action: nil, max_improvement_attempt: nil, modified_at: nil, status: nil, success_rate: nil)
|
|
68
68
|
# Some parameter documentations has been truncated, see
|
|
69
69
|
# {Deeprails::Models::DefendResponse} for more details.
|
|
70
70
|
#
|
|
@@ -76,41 +76,42 @@ module Deeprails
|
|
|
76
76
|
#
|
|
77
77
|
# @param description [String] Description for the workflow.
|
|
78
78
|
#
|
|
79
|
-
# @param improvement_action [Symbol, Deeprails::Models::DefendResponse::ImprovementAction
|
|
79
|
+
# @param improvement_action [Symbol, Deeprails::Models::DefendResponse::ImprovementAction] The action used to improve outputs that fail one or more guardrail metrics for t
|
|
80
80
|
#
|
|
81
|
-
# @param
|
|
81
|
+
# @param max_improvement_attempt [Integer] Max. number of improvement action retries until a given event passes the guardra
|
|
82
82
|
#
|
|
83
83
|
# @param modified_at [Time] The most recent time the workflow was modified in UTC.
|
|
84
84
|
#
|
|
85
|
-
# @param status [Symbol, Deeprails::Models::DefendResponse::Status] Status of the selected workflow. May be `
|
|
85
|
+
# @param status [Symbol, Deeprails::Models::DefendResponse::Status] Status of the selected workflow. May be `inactive` or `active`. Inactive workf
|
|
86
86
|
#
|
|
87
87
|
# @param success_rate [Float] Rate of events associated with this workflow that passed evaluation.
|
|
88
88
|
|
|
89
89
|
# The action used to improve outputs that fail one or more guardrail metrics for
|
|
90
|
-
# the workflow events. May be `
|
|
91
|
-
#
|
|
92
|
-
#
|
|
93
|
-
#
|
|
90
|
+
# the workflow events. May be `regen`, `fixit`, or `do_nothing`. ReGen runs the
|
|
91
|
+
# user's input prompt with minor induced variance. FixIt attempts to directly
|
|
92
|
+
# address the shortcomings of the output using the guardrail failure rationale. Do
|
|
93
|
+
# Nothing does not attempt any improvement.
|
|
94
94
|
#
|
|
95
95
|
# @see Deeprails::Models::DefendResponse#improvement_action
|
|
96
96
|
module ImprovementAction
|
|
97
97
|
extend Deeprails::Internal::Type::Enum
|
|
98
98
|
|
|
99
|
-
|
|
99
|
+
REGEN = :regen
|
|
100
100
|
FIXIT = :fixit
|
|
101
|
+
DO_NOTHING = :do_nothing
|
|
101
102
|
|
|
102
103
|
# @!method self.values
|
|
103
104
|
# @return [Array<Symbol>]
|
|
104
105
|
end
|
|
105
106
|
|
|
106
|
-
# Status of the selected workflow. May be `
|
|
107
|
+
# Status of the selected workflow. May be `inactive` or `active`. Inactive
|
|
107
108
|
# workflows will not accept events.
|
|
108
109
|
#
|
|
109
110
|
# @see Deeprails::Models::DefendResponse#status
|
|
110
111
|
module Status
|
|
111
112
|
extend Deeprails::Internal::Type::Enum
|
|
112
113
|
|
|
113
|
-
|
|
114
|
+
INACTIVE = :inactive
|
|
114
115
|
ACTIVE = :active
|
|
115
116
|
|
|
116
117
|
# @!method self.values
|
|
@@ -9,8 +9,8 @@ module Deeprails
|
|
|
9
9
|
|
|
10
10
|
# @!attribute model_input
|
|
11
11
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
12
|
-
# contain at least
|
|
13
|
-
#
|
|
12
|
+
# contain at least `user_prompt` or `system_prompt` field. For the
|
|
13
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
14
14
|
#
|
|
15
15
|
# @return [Deeprails::Models::DefendSubmitEventParams::ModelInput]
|
|
16
16
|
required :model_input, -> { Deeprails::DefendSubmitEventParams::ModelInput }
|
|
@@ -79,8 +79,8 @@ module Deeprails
|
|
|
79
79
|
|
|
80
80
|
# @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
|
|
81
81
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
82
|
-
# contain at least
|
|
83
|
-
#
|
|
82
|
+
# contain at least `user_prompt` or `system_prompt` field. For the
|
|
83
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
84
84
|
#
|
|
85
85
|
# @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
86
86
|
#
|
|
@@ -9,8 +9,8 @@ module Deeprails
|
|
|
9
9
|
|
|
10
10
|
# @!attribute model_input
|
|
11
11
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
12
|
-
# contain at least
|
|
13
|
-
#
|
|
12
|
+
# contain at least `user_prompt` or `system_prompt` field. For
|
|
13
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
14
14
|
#
|
|
15
15
|
# @return [Deeprails::Models::EvaluateCreateParams::ModelInput]
|
|
16
16
|
required :model_input, -> { Deeprails::EvaluateCreateParams::ModelInput }
|
|
@@ -90,8 +90,8 @@ module Deeprails
|
|
|
90
90
|
|
|
91
91
|
# @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
|
|
92
92
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
93
|
-
# contain at least
|
|
94
|
-
#
|
|
93
|
+
# contain at least `user_prompt` or `system_prompt` field. For
|
|
94
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
95
95
|
#
|
|
96
96
|
# @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
97
97
|
#
|
|
@@ -18,8 +18,8 @@ module Deeprails
|
|
|
18
18
|
|
|
19
19
|
# @!attribute model_input
|
|
20
20
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
21
|
-
# contain at least
|
|
22
|
-
#
|
|
21
|
+
# contain at least `user_prompt` or `system_prompt` field. For
|
|
22
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
23
23
|
#
|
|
24
24
|
# @return [Deeprails::Models::Evaluation::ModelInput]
|
|
25
25
|
required :model_input, -> { Deeprails::Evaluation::ModelInput }
|
|
@@ -189,8 +189,8 @@ module Deeprails
|
|
|
189
189
|
|
|
190
190
|
# @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
|
|
191
191
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
192
|
-
# contain at least
|
|
193
|
-
#
|
|
192
|
+
# contain at least `user_prompt` or `system_prompt` field. For
|
|
193
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
194
194
|
#
|
|
195
195
|
# @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
196
196
|
#
|
|
@@ -19,8 +19,8 @@ module Deeprails
|
|
|
19
19
|
|
|
20
20
|
# @!attribute model_input
|
|
21
21
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
22
|
-
# contain at least
|
|
23
|
-
#
|
|
22
|
+
# contain at least a `user_prompt` or `system_prompt` field. For
|
|
23
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
24
24
|
#
|
|
25
25
|
# @return [Deeprails::Models::MonitorSubmitEventParams::ModelInput]
|
|
26
26
|
required :model_input, -> { Deeprails::MonitorSubmitEventParams::ModelInput }
|
|
@@ -105,8 +105,8 @@ module Deeprails
|
|
|
105
105
|
|
|
106
106
|
# @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
|
|
107
107
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
108
|
-
# contain at least
|
|
109
|
-
#
|
|
108
|
+
# contain at least a `user_prompt` or `system_prompt` field. For
|
|
109
|
+
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
110
110
|
#
|
|
111
111
|
# @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
112
112
|
#
|
|
@@ -9,21 +9,21 @@ module Deeprails
|
|
|
9
9
|
# Use this endpoint to create a new guardrail workflow with optional guardrail
|
|
10
10
|
# thresholds and improvement actions
|
|
11
11
|
#
|
|
12
|
-
# @overload create_workflow(improvement_action:,
|
|
12
|
+
# @overload create_workflow(improvement_action:, name:, type:, automatic_hallucination_tolerance_levels: nil, custom_hallucination_threshold_values: nil, description: nil, max_improvement_attempt: nil, request_options: {})
|
|
13
13
|
#
|
|
14
|
-
# @param improvement_action [Symbol, Deeprails::Models::DefendCreateWorkflowParams::ImprovementAction
|
|
15
|
-
#
|
|
16
|
-
# @param metrics [Hash{Symbol=>Float}] Mapping of guardrail metrics to floating point threshold values. If the workflo
|
|
14
|
+
# @param improvement_action [Symbol, Deeprails::Models::DefendCreateWorkflowParams::ImprovementAction] The action used to improve outputs that fail one or guardrail metrics for the wo
|
|
17
15
|
#
|
|
18
16
|
# @param name [String] Name of the workflow.
|
|
19
17
|
#
|
|
20
18
|
# @param type [Symbol, Deeprails::Models::DefendCreateWorkflowParams::Type] Type of thresholds to use for the workflow, either `automatic` or `custom`. Aut
|
|
21
19
|
#
|
|
22
|
-
# @param
|
|
20
|
+
# @param automatic_hallucination_tolerance_levels [Hash{Symbol=>Symbol, Deeprails::Models::DefendCreateWorkflowParams::AutomaticHallucinationToleranceLevel}] Mapping of guardrail metrics to hallucination tolerance levels
|
|
21
|
+
#
|
|
22
|
+
# @param custom_hallucination_threshold_values [Hash{Symbol=>Float}] Mapping of guardrail metrics to floating point threshold values. Possible metric
|
|
23
23
|
#
|
|
24
24
|
# @param description [String] Description for the workflow.
|
|
25
25
|
#
|
|
26
|
-
# @param
|
|
26
|
+
# @param max_improvement_attempt [Integer] Max. number of improvement action retries until a given event passes the guardra
|
|
27
27
|
#
|
|
28
28
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
29
29
|
#
|
data/lib/deeprails/version.rb
CHANGED