deeprails 0.18.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +92 -0
- data/README.md +1 -1
- data/lib/deeprails/internal/stream.rb +29 -0
- data/lib/deeprails/internal/transport/base_client.rb +3 -1
- data/lib/deeprails/internal/transport/pooled_net_requester.rb +13 -11
- data/lib/deeprails/internal/type/base_stream.rb +83 -0
- data/lib/deeprails/internal/util.rb +39 -3
- data/lib/deeprails/models/defend_create_workflow_params.rb +14 -1
- data/lib/deeprails/models/defend_response.rb +134 -10
- data/lib/deeprails/models/defend_retrieve_event_params.rb +7 -1
- data/lib/deeprails/models/defend_retrieve_workflow_params.rb +8 -1
- data/lib/deeprails/models/defend_submit_and_stream_event_params.rb +92 -0
- data/lib/deeprails/models/defend_submit_and_stream_event_response.rb +7 -0
- data/lib/deeprails/models/defend_submit_event_params.rb +70 -20
- data/lib/deeprails/models/defend_update_response.rb +9 -1
- data/lib/deeprails/models/defend_update_workflow_params.rb +138 -5
- data/lib/deeprails/models/monitor_create_params.rb +14 -1
- data/lib/deeprails/models/monitor_detail_response.rb +60 -17
- data/lib/deeprails/models/monitor_event_detail_response.rb +11 -5
- data/lib/deeprails/models/monitor_retrieve_event_params.rb +7 -1
- data/lib/deeprails/models/monitor_retrieve_params.rb +8 -1
- data/lib/deeprails/models/monitor_submit_event_params.rb +70 -20
- data/lib/deeprails/models/monitor_update_params.rb +53 -5
- data/lib/deeprails/models/workflow_event_detail_response.rb +99 -16
- data/lib/deeprails/models/workflow_event_response.rb +9 -1
- data/lib/deeprails/models.rb +2 -0
- data/lib/deeprails/resources/defend.rb +70 -5
- data/lib/deeprails/resources/monitor.rb +16 -7
- data/lib/deeprails/version.rb +1 -1
- data/lib/deeprails.rb +5 -0
- data/manifest.yaml +1 -0
- data/rbi/deeprails/internal/stream.rbi +20 -0
- data/rbi/deeprails/internal/transport/base_client.rbi +18 -2
- data/rbi/deeprails/internal/type/base_stream.rbi +75 -0
- data/rbi/deeprails/internal/util.rbi +21 -1
- data/rbi/deeprails/models/defend_create_workflow_params.rbi +21 -0
- data/rbi/deeprails/models/defend_response.rbi +286 -14
- data/rbi/deeprails/models/defend_retrieve_event_params.rbi +10 -2
- data/rbi/deeprails/models/defend_retrieve_workflow_params.rbi +10 -1
- data/rbi/deeprails/models/defend_submit_and_stream_event_params.rbi +155 -0
- data/rbi/deeprails/models/defend_submit_and_stream_event_response.rbi +7 -0
- data/rbi/deeprails/models/defend_submit_event_params.rbi +134 -31
- data/rbi/deeprails/models/defend_update_response.rbi +14 -3
- data/rbi/deeprails/models/defend_update_workflow_params.rbi +304 -4
- data/rbi/deeprails/models/monitor_create_params.rbi +21 -0
- data/rbi/deeprails/models/monitor_detail_response.rbi +122 -28
- data/rbi/deeprails/models/monitor_event_detail_response.rbi +20 -8
- data/rbi/deeprails/models/monitor_retrieve_event_params.rbi +10 -2
- data/rbi/deeprails/models/monitor_retrieve_params.rbi +10 -1
- data/rbi/deeprails/models/monitor_submit_event_params.rbi +134 -31
- data/rbi/deeprails/models/monitor_update_params.rbi +113 -4
- data/rbi/deeprails/models/workflow_event_detail_response.rbi +204 -24
- data/rbi/deeprails/models/workflow_event_response.rbi +8 -0
- data/rbi/deeprails/models.rbi +3 -0
- data/rbi/deeprails/resources/defend.rbi +101 -6
- data/rbi/deeprails/resources/monitor.rbi +31 -8
- data/sig/deeprails/internal/stream.rbs +9 -0
- data/sig/deeprails/internal/type/base_stream.rbs +38 -0
- data/sig/deeprails/internal/util.rbs +10 -0
- data/sig/deeprails/models/defend_create_workflow_params.rbs +7 -0
- data/sig/deeprails/models/defend_response.rbs +114 -8
- data/sig/deeprails/models/defend_retrieve_event_params.rbs +6 -1
- data/sig/deeprails/models/defend_retrieve_workflow_params.rbs +6 -1
- data/sig/deeprails/models/defend_submit_and_stream_event_params.rbs +73 -0
- data/sig/deeprails/models/defend_submit_and_stream_event_response.rbs +5 -0
- data/sig/deeprails/models/defend_submit_event_params.rbs +53 -13
- data/sig/deeprails/models/defend_update_response.rbs +10 -3
- data/sig/deeprails/models/defend_update_workflow_params.rbs +108 -1
- data/sig/deeprails/models/monitor_create_params.rbs +7 -0
- data/sig/deeprails/models/monitor_detail_response.rbs +48 -13
- data/sig/deeprails/models/monitor_event_detail_response.rbs +12 -4
- data/sig/deeprails/models/monitor_retrieve_event_params.rbs +6 -1
- data/sig/deeprails/models/monitor_retrieve_params.rbs +6 -1
- data/sig/deeprails/models/monitor_submit_event_params.rbs +53 -13
- data/sig/deeprails/models/monitor_update_params.rbs +50 -1
- data/sig/deeprails/models/workflow_event_detail_response.rbs +100 -15
- data/sig/deeprails/models/workflow_event_response.rbs +5 -0
- data/sig/deeprails/models.rbs +2 -0
- data/sig/deeprails/resources/defend.rbs +20 -0
- data/sig/deeprails/resources/monitor.rbs +4 -0
- metadata +28 -2
|
@@ -7,6 +7,11 @@ module Deeprails
|
|
|
7
7
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
8
8
|
include Deeprails::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
|
+
# @!attribute monitor_id
|
|
11
|
+
#
|
|
12
|
+
# @return [String]
|
|
13
|
+
required :monitor_id, String
|
|
14
|
+
|
|
10
15
|
# @!attribute limit
|
|
11
16
|
# Limit the number of returned evaluations associated with this monitor. Defaults
|
|
12
17
|
# to 10.
|
|
@@ -14,10 +19,12 @@ module Deeprails
|
|
|
14
19
|
# @return [Integer, nil]
|
|
15
20
|
optional :limit, Integer
|
|
16
21
|
|
|
17
|
-
# @!method initialize(limit: nil, request_options: {})
|
|
22
|
+
# @!method initialize(monitor_id:, limit: nil, request_options: {})
|
|
18
23
|
# Some parameter documentations has been truncated, see
|
|
19
24
|
# {Deeprails::Models::MonitorRetrieveParams} for more details.
|
|
20
25
|
#
|
|
26
|
+
# @param monitor_id [String]
|
|
27
|
+
#
|
|
21
28
|
# @param limit [Integer] Limit the number of returned evaluations associated with this monitor. Defaults
|
|
22
29
|
#
|
|
23
30
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
@@ -7,10 +7,15 @@ module Deeprails
|
|
|
7
7
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
8
8
|
include Deeprails::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
|
+
# @!attribute monitor_id
|
|
11
|
+
#
|
|
12
|
+
# @return [String]
|
|
13
|
+
required :monitor_id, String
|
|
14
|
+
|
|
10
15
|
# @!attribute model_input
|
|
11
16
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
12
|
-
# contain
|
|
13
|
-
#
|
|
17
|
+
# contain a `user_prompt` field. For ground_truth_adherence guardrail metric,
|
|
18
|
+
# `ground_truth` should be provided.
|
|
14
19
|
#
|
|
15
20
|
# @return [Deeprails::Models::MonitorSubmitEventParams::ModelInput]
|
|
16
21
|
required :model_input, -> { Deeprails::MonitorSubmitEventParams::ModelInput }
|
|
@@ -30,16 +35,21 @@ module Deeprails
|
|
|
30
35
|
# @!attribute run_mode
|
|
31
36
|
# Run mode for the monitor event. The run mode allows the user to optimize for
|
|
32
37
|
# speed, accuracy, and cost by determining which models are used to evaluate the
|
|
33
|
-
# event. Available run modes
|
|
34
|
-
# `
|
|
38
|
+
# event. Available run modes (fastest to most thorough): `super_fast`, `fast`,
|
|
39
|
+
# `precision`, `precision_codex`, `precision_max`, and `precision_max_codex`.
|
|
40
|
+
# Defaults to `fast`. Note: `super_fast` does not support Web Search or File
|
|
41
|
+
# Search — if your monitor has these capabilities enabled, use a different run
|
|
42
|
+
# mode or edit the monitor to disable them.
|
|
35
43
|
#
|
|
36
44
|
# @return [Symbol, Deeprails::Models::MonitorSubmitEventParams::RunMode, nil]
|
|
37
45
|
optional :run_mode, enum: -> { Deeprails::MonitorSubmitEventParams::RunMode }
|
|
38
46
|
|
|
39
|
-
# @!method initialize(model_input:, model_output:, nametag: nil, run_mode: nil, request_options: {})
|
|
47
|
+
# @!method initialize(monitor_id:, model_input:, model_output:, nametag: nil, run_mode: nil, request_options: {})
|
|
40
48
|
# Some parameter documentations has been truncated, see
|
|
41
49
|
# {Deeprails::Models::MonitorSubmitEventParams} for more details.
|
|
42
50
|
#
|
|
51
|
+
# @param monitor_id [String]
|
|
52
|
+
#
|
|
43
53
|
# @param model_input [Deeprails::Models::MonitorSubmitEventParams::ModelInput] A dictionary of inputs sent to the LLM to generate output. The dictionary must c
|
|
44
54
|
#
|
|
45
55
|
# @param model_output [String] Output generated by the LLM to be evaluated.
|
|
@@ -51,6 +61,23 @@ module Deeprails
|
|
|
51
61
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
52
62
|
|
|
53
63
|
class ModelInput < Deeprails::Internal::Type::BaseModel
|
|
64
|
+
# @!attribute user_prompt
|
|
65
|
+
# The user prompt used to generate the output.
|
|
66
|
+
#
|
|
67
|
+
# @return [String]
|
|
68
|
+
required :user_prompt, String
|
|
69
|
+
|
|
70
|
+
# @!attribute context
|
|
71
|
+
# Any structured information that directly relates to the model’s input and
|
|
72
|
+
# expected output—e.g., the recent turn-by-turn history between an AI tutor and a
|
|
73
|
+
# student, facts or state passed through an agentic workflow, or other
|
|
74
|
+
# domain-specific signals your system already knows and wants the model to
|
|
75
|
+
# condition on.
|
|
76
|
+
#
|
|
77
|
+
# @return [Array<Deeprails::Models::MonitorSubmitEventParams::ModelInput::Context>, nil]
|
|
78
|
+
optional :context,
|
|
79
|
+
-> { Deeprails::Internal::Type::ArrayOf[Deeprails::MonitorSubmitEventParams::ModelInput::Context] }
|
|
80
|
+
|
|
54
81
|
# @!attribute ground_truth
|
|
55
82
|
# The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
56
83
|
#
|
|
@@ -63,35 +90,58 @@ module Deeprails
|
|
|
63
90
|
# @return [String, nil]
|
|
64
91
|
optional :system_prompt, String
|
|
65
92
|
|
|
66
|
-
# @!
|
|
67
|
-
#
|
|
93
|
+
# @!method initialize(user_prompt:, context: nil, ground_truth: nil, system_prompt: nil)
|
|
94
|
+
# Some parameter documentations has been truncated, see
|
|
95
|
+
# {Deeprails::Models::MonitorSubmitEventParams::ModelInput} for more details.
|
|
68
96
|
#
|
|
69
|
-
# @return [String, nil]
|
|
70
|
-
optional :user_prompt, String
|
|
71
|
-
|
|
72
|
-
# @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
|
|
73
97
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
74
|
-
# contain
|
|
75
|
-
#
|
|
98
|
+
# contain a `user_prompt` field. For ground_truth_adherence guardrail metric,
|
|
99
|
+
# `ground_truth` should be provided.
|
|
100
|
+
#
|
|
101
|
+
# @param user_prompt [String] The user prompt used to generate the output.
|
|
102
|
+
#
|
|
103
|
+
# @param context [Array<Deeprails::Models::MonitorSubmitEventParams::ModelInput::Context>] Any structured information that directly relates to the model’s input and expect
|
|
76
104
|
#
|
|
77
105
|
# @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
78
106
|
#
|
|
79
107
|
# @param system_prompt [String] The system prompt used to generate the output.
|
|
80
|
-
|
|
81
|
-
|
|
108
|
+
|
|
109
|
+
class Context < Deeprails::Internal::Type::BaseModel
|
|
110
|
+
# @!attribute content
|
|
111
|
+
# The content of the message.
|
|
112
|
+
#
|
|
113
|
+
# @return [String, nil]
|
|
114
|
+
optional :content, String
|
|
115
|
+
|
|
116
|
+
# @!attribute role
|
|
117
|
+
# The role of the speaker.
|
|
118
|
+
#
|
|
119
|
+
# @return [String, nil]
|
|
120
|
+
optional :role, String
|
|
121
|
+
|
|
122
|
+
# @!method initialize(content: nil, role: nil)
|
|
123
|
+
# @param content [String] The content of the message.
|
|
124
|
+
#
|
|
125
|
+
# @param role [String] The role of the speaker.
|
|
126
|
+
end
|
|
82
127
|
end
|
|
83
128
|
|
|
84
129
|
# Run mode for the monitor event. The run mode allows the user to optimize for
|
|
85
130
|
# speed, accuracy, and cost by determining which models are used to evaluate the
|
|
86
|
-
# event. Available run modes
|
|
87
|
-
# `
|
|
131
|
+
# event. Available run modes (fastest to most thorough): `super_fast`, `fast`,
|
|
132
|
+
# `precision`, `precision_codex`, `precision_max`, and `precision_max_codex`.
|
|
133
|
+
# Defaults to `fast`. Note: `super_fast` does not support Web Search or File
|
|
134
|
+
# Search — if your monitor has these capabilities enabled, use a different run
|
|
135
|
+
# mode or edit the monitor to disable them.
|
|
88
136
|
module RunMode
|
|
89
137
|
extend Deeprails::Internal::Type::Enum
|
|
90
138
|
|
|
91
|
-
|
|
139
|
+
SUPER_FAST = :super_fast
|
|
140
|
+
FAST = :fast
|
|
92
141
|
PRECISION = :precision
|
|
93
|
-
|
|
94
|
-
|
|
142
|
+
PRECISION_CODEX = :precision_codex
|
|
143
|
+
PRECISION_MAX = :precision_max
|
|
144
|
+
PRECISION_MAX_CODEX = :precision_max_codex
|
|
95
145
|
|
|
96
146
|
# @!method self.values
|
|
97
147
|
# @return [Array<Symbol>]
|
|
@@ -7,14 +7,34 @@ module Deeprails
|
|
|
7
7
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
8
8
|
include Deeprails::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
|
+
# @!attribute monitor_id
|
|
11
|
+
#
|
|
12
|
+
# @return [String]
|
|
13
|
+
required :monitor_id, String
|
|
14
|
+
|
|
10
15
|
# @!attribute description
|
|
11
|
-
#
|
|
16
|
+
# New description of the monitor.
|
|
12
17
|
#
|
|
13
18
|
# @return [String, nil]
|
|
14
19
|
optional :description, String
|
|
15
20
|
|
|
21
|
+
# @!attribute file_search
|
|
22
|
+
# An array of file IDs to search in the monitor's evaluations. Files must be
|
|
23
|
+
# uploaded via the DeepRails API first.
|
|
24
|
+
#
|
|
25
|
+
# @return [Array<String>, nil]
|
|
26
|
+
optional :file_search, Deeprails::Internal::Type::ArrayOf[String]
|
|
27
|
+
|
|
28
|
+
# @!attribute guardrail_metrics
|
|
29
|
+
# An array of the new guardrail metrics that model input and output pairs will be
|
|
30
|
+
# evaluated on.
|
|
31
|
+
#
|
|
32
|
+
# @return [Array<Symbol, Deeprails::Models::MonitorUpdateParams::GuardrailMetric>, nil]
|
|
33
|
+
optional :guardrail_metrics,
|
|
34
|
+
-> { Deeprails::Internal::Type::ArrayOf[enum: Deeprails::MonitorUpdateParams::GuardrailMetric] }
|
|
35
|
+
|
|
16
36
|
# @!attribute name
|
|
17
|
-
#
|
|
37
|
+
# New name of the monitor.
|
|
18
38
|
#
|
|
19
39
|
# @return [String, nil]
|
|
20
40
|
optional :name, String
|
|
@@ -26,18 +46,46 @@ module Deeprails
|
|
|
26
46
|
# @return [Symbol, Deeprails::Models::MonitorUpdateParams::Status, nil]
|
|
27
47
|
optional :status, enum: -> { Deeprails::MonitorUpdateParams::Status }
|
|
28
48
|
|
|
29
|
-
# @!
|
|
49
|
+
# @!attribute web_search
|
|
50
|
+
# Whether to enable web search for this monitor's evaluations.
|
|
51
|
+
#
|
|
52
|
+
# @return [Boolean, nil]
|
|
53
|
+
optional :web_search, Deeprails::Internal::Type::Boolean
|
|
54
|
+
|
|
55
|
+
# @!method initialize(monitor_id:, description: nil, file_search: nil, guardrail_metrics: nil, name: nil, status: nil, web_search: nil, request_options: {})
|
|
30
56
|
# Some parameter documentations has been truncated, see
|
|
31
57
|
# {Deeprails::Models::MonitorUpdateParams} for more details.
|
|
32
58
|
#
|
|
33
|
-
# @param
|
|
59
|
+
# @param monitor_id [String]
|
|
60
|
+
#
|
|
61
|
+
# @param description [String] New description of the monitor.
|
|
62
|
+
#
|
|
63
|
+
# @param file_search [Array<String>] An array of file IDs to search in the monitor's evaluations. Files must be uploa
|
|
34
64
|
#
|
|
35
|
-
# @param
|
|
65
|
+
# @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorUpdateParams::GuardrailMetric>] An array of the new guardrail metrics that model input and output pairs will be
|
|
66
|
+
#
|
|
67
|
+
# @param name [String] New name of the monitor.
|
|
36
68
|
#
|
|
37
69
|
# @param status [Symbol, Deeprails::Models::MonitorUpdateParams::Status] Status of the monitor. Can be `active` or `inactive`. Inactive monitors no lon
|
|
38
70
|
#
|
|
71
|
+
# @param web_search [Boolean] Whether to enable web search for this monitor's evaluations.
|
|
72
|
+
#
|
|
39
73
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
40
74
|
|
|
75
|
+
module GuardrailMetric
|
|
76
|
+
extend Deeprails::Internal::Type::Enum
|
|
77
|
+
|
|
78
|
+
CORRECTNESS = :correctness
|
|
79
|
+
COMPLETENESS = :completeness
|
|
80
|
+
INSTRUCTION_ADHERENCE = :instruction_adherence
|
|
81
|
+
CONTEXT_ADHERENCE = :context_adherence
|
|
82
|
+
GROUND_TRUTH_ADHERENCE = :ground_truth_adherence
|
|
83
|
+
COMPREHENSIVE_SAFETY = :comprehensive_safety
|
|
84
|
+
|
|
85
|
+
# @!method self.values
|
|
86
|
+
# @return [Array<Symbol>]
|
|
87
|
+
end
|
|
88
|
+
|
|
41
89
|
# Status of the monitor. Can be `active` or `inactive`. Inactive monitors no
|
|
42
90
|
# longer record and evaluate events.
|
|
43
91
|
module Status
|
|
@@ -4,6 +4,11 @@ module Deeprails
|
|
|
4
4
|
module Models
|
|
5
5
|
# @see Deeprails::Resources::Defend#retrieve_event
|
|
6
6
|
class WorkflowEventDetailResponse < Deeprails::Internal::Type::BaseModel
|
|
7
|
+
# @!attribute analysis_of_failures
|
|
8
|
+
#
|
|
9
|
+
# @return [Array<String>]
|
|
10
|
+
required :analysis_of_failures, Deeprails::Internal::Type::ArrayOf[String]
|
|
11
|
+
|
|
7
12
|
# @!attribute evaluation_history
|
|
8
13
|
# History of evaluations for the event.
|
|
9
14
|
#
|
|
@@ -44,13 +49,24 @@ module Deeprails
|
|
|
44
49
|
required :improvement_action, enum: -> { Deeprails::WorkflowEventDetailResponse::ImprovementAction }
|
|
45
50
|
|
|
46
51
|
# @!attribute improvement_tool_status
|
|
47
|
-
# Status of the improvement tool used to improve the event.
|
|
52
|
+
# Status of the improvement tool used to improve the event. `improvement_required`
|
|
53
|
+
# indicates that the evaluation is complete and the improvement action is needed
|
|
54
|
+
# but is not taking place. `improved` and `improvement_failed` indicate when the
|
|
55
|
+
# improvement action concludes, successfully and unsuccessfully, respectively.
|
|
56
|
+
# `no_improvement_required` means that the first evaluation passed all its
|
|
57
|
+
# metrics!
|
|
48
58
|
#
|
|
49
59
|
# @return [Symbol, Deeprails::Models::WorkflowEventDetailResponse::ImprovementToolStatus, nil]
|
|
50
60
|
required :improvement_tool_status,
|
|
51
61
|
enum: -> { Deeprails::WorkflowEventDetailResponse::ImprovementToolStatus },
|
|
52
62
|
nil?: true
|
|
53
63
|
|
|
64
|
+
# @!attribute key_improvements
|
|
65
|
+
#
|
|
66
|
+
# @return [Array<Deeprails::Models::WorkflowEventDetailResponse::KeyImprovement>]
|
|
67
|
+
required :key_improvements,
|
|
68
|
+
-> { Deeprails::Internal::Type::ArrayOf[Deeprails::WorkflowEventDetailResponse::KeyImprovement] }
|
|
69
|
+
|
|
54
70
|
# @!attribute status
|
|
55
71
|
# Status of the event.
|
|
56
72
|
#
|
|
@@ -78,8 +94,8 @@ module Deeprails
|
|
|
78
94
|
-> { Deeprails::Internal::Type::HashOf[enum: Deeprails::WorkflowEventDetailResponse::AutomaticHallucinationToleranceLevel] }
|
|
79
95
|
|
|
80
96
|
# @!attribute capabilities
|
|
81
|
-
# Extended AI capabilities available to the event, if any. Can be `web_search
|
|
82
|
-
# and/or `
|
|
97
|
+
# Extended AI capabilities available to the event, if any. Can be `web_search`,
|
|
98
|
+
# `file_search`, and/or `context_awareness`.
|
|
83
99
|
#
|
|
84
100
|
# @return [Array<Deeprails::Models::WorkflowEventDetailResponse::Capability>, nil]
|
|
85
101
|
optional :capabilities,
|
|
@@ -99,10 +115,19 @@ module Deeprails
|
|
|
99
115
|
# @return [Array<Deeprails::Models::WorkflowEventDetailResponse::File>, nil]
|
|
100
116
|
optional :files, -> { Deeprails::Internal::Type::ArrayOf[Deeprails::WorkflowEventDetailResponse::File] }
|
|
101
117
|
|
|
102
|
-
# @!
|
|
118
|
+
# @!attribute max_improvement_attempts
|
|
119
|
+
# The maximum number of improvement attempts to be applied to one event before it
|
|
120
|
+
# is considered failed.
|
|
121
|
+
#
|
|
122
|
+
# @return [Integer, nil]
|
|
123
|
+
optional :max_improvement_attempts, Integer
|
|
124
|
+
|
|
125
|
+
# @!method initialize(analysis_of_failures:, evaluation_history:, evaluation_result:, event_id:, filtered:, improved_model_output:, improvement_action:, improvement_tool_status:, key_improvements:, status:, threshold_type:, workflow_id:, automatic_hallucination_tolerance_levels: nil, capabilities: nil, custom_hallucination_threshold_values: nil, files: nil, max_improvement_attempts: nil)
|
|
103
126
|
# Some parameter documentations has been truncated, see
|
|
104
127
|
# {Deeprails::Models::WorkflowEventDetailResponse} for more details.
|
|
105
128
|
#
|
|
129
|
+
# @param analysis_of_failures [Array<String>]
|
|
130
|
+
#
|
|
106
131
|
# @param evaluation_history [Array<Deeprails::Models::WorkflowEventDetailResponse::EvaluationHistory>] History of evaluations for the event.
|
|
107
132
|
#
|
|
108
133
|
# @param evaluation_result [Hash{Symbol=>Object}] Evaluation result consisting of average scores and rationales for each of the ev
|
|
@@ -115,7 +140,9 @@ module Deeprails
|
|
|
115
140
|
#
|
|
116
141
|
# @param improvement_action [Symbol, Deeprails::Models::WorkflowEventDetailResponse::ImprovementAction] Type of improvement action used to improve the event.
|
|
117
142
|
#
|
|
118
|
-
# @param improvement_tool_status [Symbol, Deeprails::Models::WorkflowEventDetailResponse::ImprovementToolStatus, nil] Status of the improvement tool used to improve the event.
|
|
143
|
+
# @param improvement_tool_status [Symbol, Deeprails::Models::WorkflowEventDetailResponse::ImprovementToolStatus, nil] Status of the improvement tool used to improve the event. `improvement_required`
|
|
144
|
+
#
|
|
145
|
+
# @param key_improvements [Array<Deeprails::Models::WorkflowEventDetailResponse::KeyImprovement>]
|
|
119
146
|
#
|
|
120
147
|
# @param status [Symbol, Deeprails::Models::WorkflowEventDetailResponse::Status] Status of the event.
|
|
121
148
|
#
|
|
@@ -125,13 +152,20 @@ module Deeprails
|
|
|
125
152
|
#
|
|
126
153
|
# @param automatic_hallucination_tolerance_levels [Hash{Symbol=>Symbol, Deeprails::Models::WorkflowEventDetailResponse::AutomaticHallucinationToleranceLevel}] Mapping of guardrail metric names to tolerance values. Values are strings (`low`
|
|
127
154
|
#
|
|
128
|
-
# @param capabilities [Array<Deeprails::Models::WorkflowEventDetailResponse::Capability>] Extended AI capabilities available to the event, if any. Can be `web_search`
|
|
155
|
+
# @param capabilities [Array<Deeprails::Models::WorkflowEventDetailResponse::Capability>] Extended AI capabilities available to the event, if any. Can be `web_search`, `f
|
|
129
156
|
#
|
|
130
157
|
# @param custom_hallucination_threshold_values [Hash{Symbol=>Float}] Mapping of guardrail metric names to threshold values. Values are floating point
|
|
131
158
|
#
|
|
132
159
|
# @param files [Array<Deeprails::Models::WorkflowEventDetailResponse::File>] List of files available to the event, if any. Will only be present if `file_sear
|
|
160
|
+
#
|
|
161
|
+
# @param max_improvement_attempts [Integer] The maximum number of improvement attempts to be applied to one event before it
|
|
133
162
|
|
|
134
163
|
class EvaluationHistory < Deeprails::Internal::Type::BaseModel
|
|
164
|
+
# @!attribute analysis_of_failures
|
|
165
|
+
#
|
|
166
|
+
# @return [String, nil]
|
|
167
|
+
optional :analysis_of_failures, String
|
|
168
|
+
|
|
135
169
|
# @!attribute attempt
|
|
136
170
|
#
|
|
137
171
|
# @return [String, nil]
|
|
@@ -167,6 +201,17 @@ module Deeprails
|
|
|
167
201
|
# @return [Array<String>, nil]
|
|
168
202
|
optional :guardrail_metrics, Deeprails::Internal::Type::ArrayOf[String]
|
|
169
203
|
|
|
204
|
+
# @!attribute improvement_tool_status
|
|
205
|
+
#
|
|
206
|
+
# @return [Symbol, Deeprails::Models::WorkflowEventDetailResponse::EvaluationHistory::ImprovementToolStatus, nil]
|
|
207
|
+
optional :improvement_tool_status,
|
|
208
|
+
enum: -> { Deeprails::WorkflowEventDetailResponse::EvaluationHistory::ImprovementToolStatus }
|
|
209
|
+
|
|
210
|
+
# @!attribute key_improvements
|
|
211
|
+
#
|
|
212
|
+
# @return [Array<String>, nil]
|
|
213
|
+
optional :key_improvements, Deeprails::Internal::Type::ArrayOf[String]
|
|
214
|
+
|
|
170
215
|
# @!attribute model_input
|
|
171
216
|
#
|
|
172
217
|
# @return [Hash{Symbol=>Object}, nil]
|
|
@@ -177,11 +222,6 @@ module Deeprails
|
|
|
177
222
|
# @return [String, nil]
|
|
178
223
|
optional :model_output, String
|
|
179
224
|
|
|
180
|
-
# @!attribute modified_at
|
|
181
|
-
#
|
|
182
|
-
# @return [Time, nil]
|
|
183
|
-
optional :modified_at, Time
|
|
184
|
-
|
|
185
225
|
# @!attribute nametag
|
|
186
226
|
#
|
|
187
227
|
# @return [String, nil]
|
|
@@ -197,7 +237,8 @@ module Deeprails
|
|
|
197
237
|
# @return [String, nil]
|
|
198
238
|
optional :run_mode, String
|
|
199
239
|
|
|
200
|
-
# @!method initialize(attempt: nil, created_at: nil, error_message: nil, evaluation_result: nil, evaluation_status: nil, evaluation_total_cost: nil, guardrail_metrics: nil,
|
|
240
|
+
# @!method initialize(analysis_of_failures: nil, attempt: nil, created_at: nil, error_message: nil, evaluation_result: nil, evaluation_status: nil, evaluation_total_cost: nil, guardrail_metrics: nil, improvement_tool_status: nil, key_improvements: nil, model_input: nil, model_output: nil, nametag: nil, progress: nil, run_mode: nil)
|
|
241
|
+
# @param analysis_of_failures [String]
|
|
201
242
|
# @param attempt [String]
|
|
202
243
|
# @param created_at [Time]
|
|
203
244
|
# @param error_message [String]
|
|
@@ -205,12 +246,26 @@ module Deeprails
|
|
|
205
246
|
# @param evaluation_status [String]
|
|
206
247
|
# @param evaluation_total_cost [Float]
|
|
207
248
|
# @param guardrail_metrics [Array<String>]
|
|
249
|
+
# @param improvement_tool_status [Symbol, Deeprails::Models::WorkflowEventDetailResponse::EvaluationHistory::ImprovementToolStatus]
|
|
250
|
+
# @param key_improvements [Array<String>]
|
|
208
251
|
# @param model_input [Hash{Symbol=>Object}]
|
|
209
252
|
# @param model_output [String]
|
|
210
|
-
# @param modified_at [Time]
|
|
211
253
|
# @param nametag [String]
|
|
212
254
|
# @param progress [Integer]
|
|
213
255
|
# @param run_mode [String]
|
|
256
|
+
|
|
257
|
+
# @see Deeprails::Models::WorkflowEventDetailResponse::EvaluationHistory#improvement_tool_status
|
|
258
|
+
module ImprovementToolStatus
|
|
259
|
+
extend Deeprails::Internal::Type::Enum
|
|
260
|
+
|
|
261
|
+
IMPROVED = :improved
|
|
262
|
+
IMPROVEMENT_FAILED = :improvement_failed
|
|
263
|
+
NO_IMPROVEMENT_REQUIRED = :no_improvement_required
|
|
264
|
+
IMPROVEMENT_REQUIRED = :improvement_required
|
|
265
|
+
|
|
266
|
+
# @!method self.values
|
|
267
|
+
# @return [Array<Symbol>]
|
|
268
|
+
end
|
|
214
269
|
end
|
|
215
270
|
|
|
216
271
|
# Type of improvement action used to improve the event.
|
|
@@ -227,20 +282,36 @@ module Deeprails
|
|
|
227
282
|
# @return [Array<Symbol>]
|
|
228
283
|
end
|
|
229
284
|
|
|
230
|
-
# Status of the improvement tool used to improve the event.
|
|
285
|
+
# Status of the improvement tool used to improve the event. `improvement_required`
|
|
286
|
+
# indicates that the evaluation is complete and the improvement action is needed
|
|
287
|
+
# but is not taking place. `improved` and `improvement_failed` indicate when the
|
|
288
|
+
# improvement action concludes, successfully and unsuccessfully, respectively.
|
|
289
|
+
# `no_improvement_required` means that the first evaluation passed all its
|
|
290
|
+
# metrics!
|
|
231
291
|
#
|
|
232
292
|
# @see Deeprails::Models::WorkflowEventDetailResponse#improvement_tool_status
|
|
233
293
|
module ImprovementToolStatus
|
|
234
294
|
extend Deeprails::Internal::Type::Enum
|
|
235
295
|
|
|
236
296
|
IMPROVED = :improved
|
|
237
|
-
|
|
297
|
+
IMPROVEMENT_FAILED = :improvement_failed
|
|
298
|
+
NO_IMPROVEMENT_REQUIRED = :no_improvement_required
|
|
238
299
|
IMPROVEMENT_REQUIRED = :improvement_required
|
|
239
300
|
|
|
240
301
|
# @!method self.values
|
|
241
302
|
# @return [Array<Symbol>]
|
|
242
303
|
end
|
|
243
304
|
|
|
305
|
+
class KeyImprovement < Deeprails::Internal::Type::BaseModel
|
|
306
|
+
# @!attribute key_improvement
|
|
307
|
+
#
|
|
308
|
+
# @return [Array<String>, nil]
|
|
309
|
+
optional :key_improvement, Deeprails::Internal::Type::ArrayOf[String]
|
|
310
|
+
|
|
311
|
+
# @!method initialize(key_improvement: nil)
|
|
312
|
+
# @param key_improvement [Array<String>]
|
|
313
|
+
end
|
|
314
|
+
|
|
244
315
|
# Status of the event.
|
|
245
316
|
#
|
|
246
317
|
# @see Deeprails::Models::WorkflowEventDetailResponse#status
|
|
@@ -304,10 +375,22 @@ module Deeprails
|
|
|
304
375
|
# @return [Integer, nil]
|
|
305
376
|
optional :file_size, Integer
|
|
306
377
|
|
|
307
|
-
# @!
|
|
378
|
+
# @!attribute presigned_url
|
|
379
|
+
#
|
|
380
|
+
# @return [String, nil]
|
|
381
|
+
optional :presigned_url, String
|
|
382
|
+
|
|
383
|
+
# @!attribute presigned_url_expires_at
|
|
384
|
+
#
|
|
385
|
+
# @return [Time, nil]
|
|
386
|
+
optional :presigned_url_expires_at, Time
|
|
387
|
+
|
|
388
|
+
# @!method initialize(file_id: nil, file_name: nil, file_size: nil, presigned_url: nil, presigned_url_expires_at: nil)
|
|
308
389
|
# @param file_id [String]
|
|
309
390
|
# @param file_name [String]
|
|
310
391
|
# @param file_size [Integer]
|
|
392
|
+
# @param presigned_url [String]
|
|
393
|
+
# @param presigned_url_expires_at [Time]
|
|
311
394
|
end
|
|
312
395
|
end
|
|
313
396
|
end
|
|
@@ -4,6 +4,12 @@ module Deeprails
|
|
|
4
4
|
module Models
|
|
5
5
|
# @see Deeprails::Resources::Defend#submit_event
|
|
6
6
|
class WorkflowEventResponse < Deeprails::Internal::Type::BaseModel
|
|
7
|
+
# @!attribute billing_request_id
|
|
8
|
+
# The ID of the billing request for the event.
|
|
9
|
+
#
|
|
10
|
+
# @return [String]
|
|
11
|
+
required :billing_request_id, String
|
|
12
|
+
|
|
7
13
|
# @!attribute created_at
|
|
8
14
|
# The time the event was created in UTC.
|
|
9
15
|
#
|
|
@@ -28,7 +34,9 @@ module Deeprails
|
|
|
28
34
|
# @return [String]
|
|
29
35
|
required :workflow_id, String
|
|
30
36
|
|
|
31
|
-
# @!method initialize(created_at:, event_id:, status:, workflow_id:)
|
|
37
|
+
# @!method initialize(billing_request_id:, created_at:, event_id:, status:, workflow_id:)
|
|
38
|
+
# @param billing_request_id [String] The ID of the billing request for the event.
|
|
39
|
+
#
|
|
32
40
|
# @param created_at [Time] The time the event was created in UTC.
|
|
33
41
|
#
|
|
34
42
|
# @param event_id [String] A unique workflow event ID.
|
data/lib/deeprails/models.rb
CHANGED
|
@@ -49,6 +49,8 @@ module Deeprails
|
|
|
49
49
|
|
|
50
50
|
DefendRetrieveWorkflowParams = Deeprails::Models::DefendRetrieveWorkflowParams
|
|
51
51
|
|
|
52
|
+
DefendSubmitAndStreamEventParams = Deeprails::Models::DefendSubmitAndStreamEventParams
|
|
53
|
+
|
|
52
54
|
DefendSubmitEventParams = Deeprails::Models::DefendSubmitEventParams
|
|
53
55
|
|
|
54
56
|
DefendUpdateResponse = Deeprails::Models::DefendUpdateResponse
|
|
@@ -9,7 +9,7 @@ module Deeprails
|
|
|
9
9
|
# Use this endpoint to create a new guardrail workflow by specifying guardrail
|
|
10
10
|
# thresholds, an improvement action, and optional extended capabilities.
|
|
11
11
|
#
|
|
12
|
-
# @overload create_workflow(improvement_action:, name:, threshold_type:, automatic_hallucination_tolerance_levels: nil, custom_hallucination_threshold_values: nil, description: nil, file_search: nil, max_improvement_attempts: nil, web_search: nil, request_options: {})
|
|
12
|
+
# @overload create_workflow(improvement_action:, name:, threshold_type:, automatic_hallucination_tolerance_levels: nil, context_awareness: nil, custom_hallucination_threshold_values: nil, description: nil, file_search: nil, max_improvement_attempts: nil, web_search: nil, request_options: {})
|
|
13
13
|
#
|
|
14
14
|
# @param improvement_action [Symbol, Deeprails::Models::DefendCreateWorkflowParams::ImprovementAction] The action used to improve outputs that fail one or more guardrail metrics for t
|
|
15
15
|
#
|
|
@@ -19,6 +19,8 @@ module Deeprails
|
|
|
19
19
|
#
|
|
20
20
|
# @param automatic_hallucination_tolerance_levels [Hash{Symbol=>Symbol, Deeprails::Models::DefendCreateWorkflowParams::AutomaticHallucinationToleranceLevel}] Mapping of guardrail metrics to hallucination tolerance levels
|
|
21
21
|
#
|
|
22
|
+
# @param context_awareness [Boolean] Context includes any structured information that directly relates to the model’s
|
|
23
|
+
#
|
|
22
24
|
# @param custom_hallucination_threshold_values [Hash{Symbol=>Float}] Mapping of guardrail metrics to floating point threshold values. Possible metric
|
|
23
25
|
#
|
|
24
26
|
# @param description [String] Description for the workflow.
|
|
@@ -90,15 +92,59 @@ module Deeprails
|
|
|
90
92
|
# @see Deeprails::Models::DefendRetrieveWorkflowParams
|
|
91
93
|
def retrieve_workflow(workflow_id, params = {})
|
|
92
94
|
parsed, options = Deeprails::DefendRetrieveWorkflowParams.dump_request(params)
|
|
95
|
+
query = Deeprails::Internal::Util.encode_query_params(parsed)
|
|
93
96
|
@client.request(
|
|
94
97
|
method: :get,
|
|
95
98
|
path: ["defend/%1$s", workflow_id],
|
|
96
|
-
query:
|
|
99
|
+
query: query,
|
|
97
100
|
model: Deeprails::DefendResponse,
|
|
98
101
|
options: options
|
|
99
102
|
)
|
|
100
103
|
end
|
|
101
104
|
|
|
105
|
+
# Some parameter documentations has been truncated, see
|
|
106
|
+
# {Deeprails::Models::DefendSubmitAndStreamEventParams} for more details.
|
|
107
|
+
#
|
|
108
|
+
# Use this endpoint to submit a model input and output pair to a workflow for
|
|
109
|
+
# evaluation with streaming responses.
|
|
110
|
+
#
|
|
111
|
+
# @overload submit_and_stream_event_streaming(workflow_id, model_input:, model_output:, model_used:, run_mode:, stream: nil, nametag: nil, request_options: {})
|
|
112
|
+
#
|
|
113
|
+
# @param workflow_id [String] Path param: The ID of the workflow to create the event for.
|
|
114
|
+
#
|
|
115
|
+
# @param model_input [Hash{Symbol=>Object}] Body param: The input provided to the model (e.g., prompt, messages).
|
|
116
|
+
#
|
|
117
|
+
# @param model_output [String] Body param: The output generated by the model to be evaluated.
|
|
118
|
+
#
|
|
119
|
+
# @param model_used [String] Body param: The model that generated the output (e.g., "gpt-4", "claude-3").
|
|
120
|
+
#
|
|
121
|
+
# @param run_mode [Symbol, Deeprails::Models::DefendSubmitAndStreamEventParams::RunMode] Body param: The evaluation run mode. Streaming is supported on all run modes exc
|
|
122
|
+
#
|
|
123
|
+
# @param stream [Boolean] Query param: Enable SSE streaming for real-time token feedback. Supported on all
|
|
124
|
+
#
|
|
125
|
+
# @param nametag [String] Body param: Optional tag to identify this event.
|
|
126
|
+
#
|
|
127
|
+
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
128
|
+
#
|
|
129
|
+
# @return [Deeprails::Internal::Stream<String>]
|
|
130
|
+
#
|
|
131
|
+
# @see Deeprails::Models::DefendSubmitAndStreamEventParams
|
|
132
|
+
def submit_and_stream_event_streaming(workflow_id, params)
|
|
133
|
+
query_params = [:stream]
|
|
134
|
+
parsed, options = Deeprails::DefendSubmitAndStreamEventParams.dump_request(params)
|
|
135
|
+
query = Deeprails::Internal::Util.encode_query_params(parsed.slice(*query_params))
|
|
136
|
+
@client.request(
|
|
137
|
+
method: :post,
|
|
138
|
+
path: ["defend/%1$s/events?stream=true", workflow_id],
|
|
139
|
+
query: query,
|
|
140
|
+
headers: {"accept" => "text/event-stream"},
|
|
141
|
+
body: parsed.except(*query_params),
|
|
142
|
+
stream: Deeprails::Internal::Stream,
|
|
143
|
+
model: String,
|
|
144
|
+
options: options
|
|
145
|
+
)
|
|
146
|
+
end
|
|
147
|
+
|
|
102
148
|
# Some parameter documentations has been truncated, see
|
|
103
149
|
# {Deeprails::Models::DefendSubmitEventParams} for more details.
|
|
104
150
|
#
|
|
@@ -135,15 +181,34 @@ module Deeprails
|
|
|
135
181
|
)
|
|
136
182
|
end
|
|
137
183
|
|
|
184
|
+
# Some parameter documentations has been truncated, see
|
|
185
|
+
# {Deeprails::Models::DefendUpdateWorkflowParams} for more details.
|
|
186
|
+
#
|
|
138
187
|
# Use this endpoint to update an existing defend workflow if its details change.
|
|
139
188
|
#
|
|
140
|
-
# @overload update_workflow(workflow_id, description: nil, name: nil, request_options: {})
|
|
189
|
+
# @overload update_workflow(workflow_id, automatic_hallucination_tolerance_levels: nil, context_awareness: nil, custom_hallucination_threshold_values: nil, description: nil, file_search: nil, improvement_action: nil, max_improvement_attempts: nil, name: nil, threshold_type: nil, web_search: nil, request_options: {})
|
|
141
190
|
#
|
|
142
191
|
# @param workflow_id [String] The ID of the workflow to edit.
|
|
143
192
|
#
|
|
144
|
-
# @param
|
|
193
|
+
# @param automatic_hallucination_tolerance_levels [Hash{Symbol=>Symbol, Deeprails::Models::DefendUpdateWorkflowParams::AutomaticHallucinationToleranceLevel}] New mapping of guardrail metrics to hallucination tolerance levels
|
|
145
194
|
#
|
|
146
|
-
# @param
|
|
195
|
+
# @param context_awareness [Boolean] Whether to enable context awareness for this workflow's evaluations.
|
|
196
|
+
#
|
|
197
|
+
# @param custom_hallucination_threshold_values [Hash{Symbol=>Float}] New mapping of guardrail metrics to floating point threshold values to be used w
|
|
198
|
+
#
|
|
199
|
+
# @param description [String] New description for the workflow.
|
|
200
|
+
#
|
|
201
|
+
# @param file_search [Array<String>] An array of file IDs to search in the workflow's evaluations. Files must be uplo
|
|
202
|
+
#
|
|
203
|
+
# @param improvement_action [Symbol, Deeprails::Models::DefendUpdateWorkflowParams::ImprovementAction] The new action used to improve outputs that fail one or more guardrail metrics f
|
|
204
|
+
#
|
|
205
|
+
# @param max_improvement_attempts [Integer] Max. number of improvement action attempts until a given event passes the guardr
|
|
206
|
+
#
|
|
207
|
+
# @param name [String] New name for the workflow.
|
|
208
|
+
#
|
|
209
|
+
# @param threshold_type [Symbol, Deeprails::Models::DefendUpdateWorkflowParams::ThresholdType] New type of thresholds to use for the workflow, either `automatic` or `custom`.
|
|
210
|
+
#
|
|
211
|
+
# @param web_search [Boolean] Whether to enable web search for this workflow's evaluations.
|
|
147
212
|
#
|
|
148
213
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
149
214
|
#
|