deeprails 0.18.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +62 -0
- data/README.md +1 -1
- data/lib/deeprails/internal/stream.rb +29 -0
- data/lib/deeprails/internal/transport/base_client.rb +3 -1
- data/lib/deeprails/internal/transport/pooled_net_requester.rb +12 -10
- data/lib/deeprails/internal/type/base_stream.rb +83 -0
- data/lib/deeprails/internal/util.rb +7 -2
- data/lib/deeprails/models/defend_create_workflow_params.rb +14 -1
- data/lib/deeprails/models/defend_response.rb +134 -10
- data/lib/deeprails/models/defend_submit_and_stream_event_params.rb +83 -0
- data/lib/deeprails/models/defend_submit_and_stream_event_response.rb +7 -0
- data/lib/deeprails/models/defend_submit_event_params.rb +52 -16
- data/lib/deeprails/models/defend_update_response.rb +9 -1
- data/lib/deeprails/models/defend_update_workflow_params.rb +131 -5
- data/lib/deeprails/models/monitor_create_params.rb +14 -1
- data/lib/deeprails/models/monitor_detail_response.rb +53 -14
- data/lib/deeprails/models/monitor_event_detail_response.rb +6 -2
- data/lib/deeprails/models/monitor_submit_event_params.rb +52 -16
- data/lib/deeprails/models/monitor_update_params.rb +46 -5
- data/lib/deeprails/models/workflow_event_detail_response.rb +88 -16
- data/lib/deeprails/models/workflow_event_response.rb +9 -1
- data/lib/deeprails/models.rb +2 -0
- data/lib/deeprails/resources/defend.rb +67 -4
- data/lib/deeprails/resources/monitor.rb +14 -6
- data/lib/deeprails/version.rb +1 -1
- data/lib/deeprails.rb +5 -0
- data/manifest.yaml +1 -0
- data/rbi/deeprails/internal/stream.rbi +20 -0
- data/rbi/deeprails/internal/transport/base_client.rbi +18 -2
- data/rbi/deeprails/internal/type/base_stream.rbi +75 -0
- data/rbi/deeprails/models/defend_create_workflow_params.rbi +21 -0
- data/rbi/deeprails/models/defend_response.rbi +286 -14
- data/rbi/deeprails/models/defend_submit_and_stream_event_params.rbi +148 -0
- data/rbi/deeprails/models/defend_submit_and_stream_event_response.rbi +7 -0
- data/rbi/deeprails/models/defend_submit_event_params.rbi +108 -25
- data/rbi/deeprails/models/defend_update_response.rbi +14 -3
- data/rbi/deeprails/models/defend_update_workflow_params.rbi +298 -4
- data/rbi/deeprails/models/monitor_create_params.rbi +21 -0
- data/rbi/deeprails/models/monitor_detail_response.rbi +103 -22
- data/rbi/deeprails/models/monitor_event_detail_response.rbi +4 -2
- data/rbi/deeprails/models/monitor_submit_event_params.rbi +108 -25
- data/rbi/deeprails/models/monitor_update_params.rbi +107 -4
- data/rbi/deeprails/models/workflow_event_detail_response.rbi +170 -24
- data/rbi/deeprails/models/workflow_event_response.rbi +8 -0
- data/rbi/deeprails/models.rbi +3 -0
- data/rbi/deeprails/resources/defend.rbi +96 -6
- data/rbi/deeprails/resources/monitor.rbi +28 -8
- data/sig/deeprails/internal/stream.rbs +9 -0
- data/sig/deeprails/internal/type/base_stream.rbs +38 -0
- data/sig/deeprails/models/defend_create_workflow_params.rbs +7 -0
- data/sig/deeprails/models/defend_response.rbs +114 -8
- data/sig/deeprails/models/defend_submit_and_stream_event_params.rbs +74 -0
- data/sig/deeprails/models/defend_submit_and_stream_event_response.rbs +5 -0
- data/sig/deeprails/models/defend_submit_event_params.rbs +39 -10
- data/sig/deeprails/models/defend_update_response.rbs +10 -3
- data/sig/deeprails/models/defend_update_workflow_params.rbs +103 -1
- data/sig/deeprails/models/monitor_create_params.rbs +7 -0
- data/sig/deeprails/models/monitor_detail_response.rbs +36 -9
- data/sig/deeprails/models/monitor_submit_event_params.rbs +39 -10
- data/sig/deeprails/models/monitor_update_params.rbs +45 -1
- data/sig/deeprails/models/workflow_event_detail_response.rbs +88 -15
- data/sig/deeprails/models/workflow_event_response.rbs +5 -0
- data/sig/deeprails/models.rbs +2 -0
- data/sig/deeprails/resources/defend.rbs +20 -0
- data/sig/deeprails/resources/monitor.rbs +4 -0
- metadata +28 -2
|
@@ -9,8 +9,8 @@ module Deeprails
|
|
|
9
9
|
|
|
10
10
|
# @!attribute model_input
|
|
11
11
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
12
|
-
# contain
|
|
13
|
-
#
|
|
12
|
+
# contain a `user_prompt` field. For the ground_truth_adherence guardrail metric,
|
|
13
|
+
# `ground_truth` should be provided.
|
|
14
14
|
#
|
|
15
15
|
# @return [Deeprails::Models::DefendSubmitEventParams::ModelInput]
|
|
16
16
|
required :model_input, -> { Deeprails::DefendSubmitEventParams::ModelInput }
|
|
@@ -30,8 +30,8 @@ module Deeprails
|
|
|
30
30
|
# @!attribute run_mode
|
|
31
31
|
# Run mode for the workflow event. The run mode allows the user to optimize for
|
|
32
32
|
# speed, accuracy, and cost by determining which models are used to evaluate the
|
|
33
|
-
# event. Available run modes include `
|
|
34
|
-
# `economy`. Defaults to `smart`.
|
|
33
|
+
# event. Available run modes include `precision_plus_codex`, `precision_plus`,
|
|
34
|
+
# `precision`, `smart`, and `economy`. Defaults to `smart`.
|
|
35
35
|
#
|
|
36
36
|
# @return [Symbol, Deeprails::Models::DefendSubmitEventParams::RunMode]
|
|
37
37
|
required :run_mode, enum: -> { Deeprails::DefendSubmitEventParams::RunMode }
|
|
@@ -59,6 +59,23 @@ module Deeprails
|
|
|
59
59
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
60
60
|
|
|
61
61
|
class ModelInput < Deeprails::Internal::Type::BaseModel
|
|
62
|
+
# @!attribute user_prompt
|
|
63
|
+
# The user prompt used to generate the output.
|
|
64
|
+
#
|
|
65
|
+
# @return [String]
|
|
66
|
+
required :user_prompt, String
|
|
67
|
+
|
|
68
|
+
# @!attribute context
|
|
69
|
+
# Any structured information that directly relates to the model’s input and
|
|
70
|
+
# expected output—e.g., the recent turn-by-turn history between an AI tutor and a
|
|
71
|
+
# student, facts or state passed through an agentic workflow, or other
|
|
72
|
+
# domain-specific signals your system already knows and wants the model to
|
|
73
|
+
# condition on.
|
|
74
|
+
#
|
|
75
|
+
# @return [Array<Deeprails::Models::DefendSubmitEventParams::ModelInput::Context>, nil]
|
|
76
|
+
optional :context,
|
|
77
|
+
-> { Deeprails::Internal::Type::ArrayOf[Deeprails::DefendSubmitEventParams::ModelInput::Context] }
|
|
78
|
+
|
|
62
79
|
# @!attribute ground_truth
|
|
63
80
|
# The ground truth for evaluating the Ground Truth Adherence guardrail.
|
|
64
81
|
#
|
|
@@ -71,31 +88,50 @@ module Deeprails
|
|
|
71
88
|
# @return [String, nil]
|
|
72
89
|
optional :system_prompt, String
|
|
73
90
|
|
|
74
|
-
# @!
|
|
75
|
-
#
|
|
91
|
+
# @!method initialize(user_prompt:, context: nil, ground_truth: nil, system_prompt: nil)
|
|
92
|
+
# Some parameter documentations has been truncated, see
|
|
93
|
+
# {Deeprails::Models::DefendSubmitEventParams::ModelInput} for more details.
|
|
76
94
|
#
|
|
77
|
-
# @return [String, nil]
|
|
78
|
-
optional :user_prompt, String
|
|
79
|
-
|
|
80
|
-
# @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
|
|
81
95
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
82
|
-
# contain
|
|
83
|
-
#
|
|
96
|
+
# contain a `user_prompt` field. For the ground_truth_adherence guardrail metric,
|
|
97
|
+
# `ground_truth` should be provided.
|
|
98
|
+
#
|
|
99
|
+
# @param user_prompt [String] The user prompt used to generate the output.
|
|
100
|
+
#
|
|
101
|
+
# @param context [Array<Deeprails::Models::DefendSubmitEventParams::ModelInput::Context>] Any structured information that directly relates to the model’s input and expect
|
|
84
102
|
#
|
|
85
103
|
# @param ground_truth [String] The ground truth for evaluating the Ground Truth Adherence guardrail.
|
|
86
104
|
#
|
|
87
105
|
# @param system_prompt [String] The system prompt used to generate the output.
|
|
88
|
-
|
|
89
|
-
|
|
106
|
+
|
|
107
|
+
class Context < Deeprails::Internal::Type::BaseModel
|
|
108
|
+
# @!attribute content
|
|
109
|
+
# The content of the message.
|
|
110
|
+
#
|
|
111
|
+
# @return [String, nil]
|
|
112
|
+
optional :content, String
|
|
113
|
+
|
|
114
|
+
# @!attribute role
|
|
115
|
+
# The role of the speaker.
|
|
116
|
+
#
|
|
117
|
+
# @return [String, nil]
|
|
118
|
+
optional :role, String
|
|
119
|
+
|
|
120
|
+
# @!method initialize(content: nil, role: nil)
|
|
121
|
+
# @param content [String] The content of the message.
|
|
122
|
+
#
|
|
123
|
+
# @param role [String] The role of the speaker.
|
|
124
|
+
end
|
|
90
125
|
end
|
|
91
126
|
|
|
92
127
|
# Run mode for the workflow event. The run mode allows the user to optimize for
|
|
93
128
|
# speed, accuracy, and cost by determining which models are used to evaluate the
|
|
94
|
-
# event. Available run modes include `
|
|
95
|
-
# `economy`. Defaults to `smart`.
|
|
129
|
+
# event. Available run modes include `precision_plus_codex`, `precision_plus`,
|
|
130
|
+
# `precision`, `smart`, and `economy`. Defaults to `smart`.
|
|
96
131
|
module RunMode
|
|
97
132
|
extend Deeprails::Internal::Type::Enum
|
|
98
133
|
|
|
134
|
+
PRECISION_PLUS_CODEX = :precision_plus_codex
|
|
99
135
|
PRECISION_PLUS = :precision_plus
|
|
100
136
|
PRECISION = :precision
|
|
101
137
|
SMART = :smart
|
|
@@ -23,7 +23,13 @@ module Deeprails
|
|
|
23
23
|
# @return [String]
|
|
24
24
|
required :workflow_id, String
|
|
25
25
|
|
|
26
|
-
# @!
|
|
26
|
+
# @!attribute name
|
|
27
|
+
# The name of the workflow.
|
|
28
|
+
#
|
|
29
|
+
# @return [String, nil]
|
|
30
|
+
optional :name, String
|
|
31
|
+
|
|
32
|
+
# @!method initialize(modified_at:, status:, workflow_id:, name: nil)
|
|
27
33
|
# Some parameter documentations has been truncated, see
|
|
28
34
|
# {Deeprails::Models::DefendUpdateResponse} for more details.
|
|
29
35
|
#
|
|
@@ -32,6 +38,8 @@ module Deeprails
|
|
|
32
38
|
# @param status [Symbol, Deeprails::Models::DefendUpdateResponse::Status] Status of the selected workflow. May be `inactive` or `active`. Inactive workf
|
|
33
39
|
#
|
|
34
40
|
# @param workflow_id [String] A unique workflow ID.
|
|
41
|
+
#
|
|
42
|
+
# @param name [String] The name of the workflow.
|
|
35
43
|
|
|
36
44
|
# Status of the selected workflow. May be `inactive` or `active`. Inactive
|
|
37
45
|
# workflows will not accept events.
|
|
@@ -7,24 +7,150 @@ module Deeprails
|
|
|
7
7
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
8
8
|
include Deeprails::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
|
+
# @!attribute automatic_hallucination_tolerance_levels
|
|
11
|
+
# New mapping of guardrail metrics to hallucination tolerance levels (either
|
|
12
|
+
# `low`, `medium`, or `high`) to be used when `threshold_type` is set to
|
|
13
|
+
# `automatic`. Possible metrics are `completeness`, `instruction_adherence`,
|
|
14
|
+
# `context_adherence`, `ground_truth_adherence`, or `comprehensive_safety`.
|
|
15
|
+
#
|
|
16
|
+
# @return [Hash{Symbol=>Symbol, Deeprails::Models::DefendUpdateWorkflowParams::AutomaticHallucinationToleranceLevel}, nil]
|
|
17
|
+
optional :automatic_hallucination_tolerance_levels,
|
|
18
|
+
-> { Deeprails::Internal::Type::HashOf[enum: Deeprails::DefendUpdateWorkflowParams::AutomaticHallucinationToleranceLevel] }
|
|
19
|
+
|
|
20
|
+
# @!attribute context_awareness
|
|
21
|
+
# Whether to enable context awareness for this workflow's evaluations.
|
|
22
|
+
#
|
|
23
|
+
# @return [Boolean, nil]
|
|
24
|
+
optional :context_awareness, Deeprails::Internal::Type::Boolean
|
|
25
|
+
|
|
26
|
+
# @!attribute custom_hallucination_threshold_values
|
|
27
|
+
# New mapping of guardrail metrics to floating point threshold values to be used
|
|
28
|
+
# when `threshold_type` is set to `custom`. Possible metrics are `correctness`,
|
|
29
|
+
# `completeness`, `instruction_adherence`, `context_adherence`,
|
|
30
|
+
# `ground_truth_adherence`, or `comprehensive_safety`.
|
|
31
|
+
#
|
|
32
|
+
# @return [Hash{Symbol=>Float}, nil]
|
|
33
|
+
optional :custom_hallucination_threshold_values, Deeprails::Internal::Type::HashOf[Float]
|
|
34
|
+
|
|
10
35
|
# @!attribute description
|
|
11
|
-
#
|
|
36
|
+
# New description for the workflow.
|
|
12
37
|
#
|
|
13
38
|
# @return [String, nil]
|
|
14
39
|
optional :description, String
|
|
15
40
|
|
|
41
|
+
# @!attribute file_search
|
|
42
|
+
# An array of file IDs to search in the workflow's evaluations. Files must be
|
|
43
|
+
# uploaded via the DeepRails API first.
|
|
44
|
+
#
|
|
45
|
+
# @return [Array<String>, nil]
|
|
46
|
+
optional :file_search, Deeprails::Internal::Type::ArrayOf[String]
|
|
47
|
+
|
|
48
|
+
# @!attribute improvement_action
|
|
49
|
+
# The new action used to improve outputs that fail one or more guardrail metrics
|
|
50
|
+
# for the workflow events. May be `regen`, `fixit`, or `do_nothing`. ReGen runs
|
|
51
|
+
# the user's input prompt with minor induced variance. FixIt attempts to directly
|
|
52
|
+
# address the shortcomings of the output using the guardrail failure rationale. Do
|
|
53
|
+
# Nothing does not attempt any improvement.
|
|
54
|
+
#
|
|
55
|
+
# @return [Symbol, Deeprails::Models::DefendUpdateWorkflowParams::ImprovementAction, nil]
|
|
56
|
+
optional :improvement_action, enum: -> { Deeprails::DefendUpdateWorkflowParams::ImprovementAction }
|
|
57
|
+
|
|
58
|
+
# @!attribute max_improvement_attempts
|
|
59
|
+
# Max. number of improvement action attempts until a given event passes the
|
|
60
|
+
# guardrails. Defaults to 10.
|
|
61
|
+
#
|
|
62
|
+
# @return [Integer, nil]
|
|
63
|
+
optional :max_improvement_attempts, Integer
|
|
64
|
+
|
|
16
65
|
# @!attribute name
|
|
17
|
-
#
|
|
66
|
+
# New name for the workflow.
|
|
18
67
|
#
|
|
19
68
|
# @return [String, nil]
|
|
20
69
|
optional :name, String
|
|
21
70
|
|
|
22
|
-
# @!
|
|
23
|
-
#
|
|
71
|
+
# @!attribute threshold_type
|
|
72
|
+
# New type of thresholds to use for the workflow, either `automatic` or `custom`.
|
|
73
|
+
# Automatic thresholds are assigned internally after the user specifies a
|
|
74
|
+
# qualitative tolerance for the metrics, whereas custom metrics allow the user to
|
|
75
|
+
# set the threshold for each metric as a floating point number between 0.0 and
|
|
76
|
+
# 1.0.
|
|
77
|
+
#
|
|
78
|
+
# @return [Symbol, Deeprails::Models::DefendUpdateWorkflowParams::ThresholdType, nil]
|
|
79
|
+
optional :threshold_type, enum: -> { Deeprails::DefendUpdateWorkflowParams::ThresholdType }
|
|
80
|
+
|
|
81
|
+
# @!attribute web_search
|
|
82
|
+
# Whether to enable web search for this workflow's evaluations.
|
|
83
|
+
#
|
|
84
|
+
# @return [Boolean, nil]
|
|
85
|
+
optional :web_search, Deeprails::Internal::Type::Boolean
|
|
86
|
+
|
|
87
|
+
# @!method initialize(automatic_hallucination_tolerance_levels: nil, context_awareness: nil, custom_hallucination_threshold_values: nil, description: nil, file_search: nil, improvement_action: nil, max_improvement_attempts: nil, name: nil, threshold_type: nil, web_search: nil, request_options: {})
|
|
88
|
+
# Some parameter documentations has been truncated, see
|
|
89
|
+
# {Deeprails::Models::DefendUpdateWorkflowParams} for more details.
|
|
90
|
+
#
|
|
91
|
+
# @param automatic_hallucination_tolerance_levels [Hash{Symbol=>Symbol, Deeprails::Models::DefendUpdateWorkflowParams::AutomaticHallucinationToleranceLevel}] New mapping of guardrail metrics to hallucination tolerance levels
|
|
24
92
|
#
|
|
25
|
-
# @param
|
|
93
|
+
# @param context_awareness [Boolean] Whether to enable context awareness for this workflow's evaluations.
|
|
94
|
+
#
|
|
95
|
+
# @param custom_hallucination_threshold_values [Hash{Symbol=>Float}] New mapping of guardrail metrics to floating point threshold values to be used w
|
|
96
|
+
#
|
|
97
|
+
# @param description [String] New description for the workflow.
|
|
98
|
+
#
|
|
99
|
+
# @param file_search [Array<String>] An array of file IDs to search in the workflow's evaluations. Files must be uplo
|
|
100
|
+
#
|
|
101
|
+
# @param improvement_action [Symbol, Deeprails::Models::DefendUpdateWorkflowParams::ImprovementAction] The new action used to improve outputs that fail one or more guardrail metrics f
|
|
102
|
+
#
|
|
103
|
+
# @param max_improvement_attempts [Integer] Max. number of improvement action attempts until a given event passes the guardr
|
|
104
|
+
#
|
|
105
|
+
# @param name [String] New name for the workflow.
|
|
106
|
+
#
|
|
107
|
+
# @param threshold_type [Symbol, Deeprails::Models::DefendUpdateWorkflowParams::ThresholdType] New type of thresholds to use for the workflow, either `automatic` or `custom`.
|
|
108
|
+
#
|
|
109
|
+
# @param web_search [Boolean] Whether to enable web search for this workflow's evaluations.
|
|
26
110
|
#
|
|
27
111
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
112
|
+
|
|
113
|
+
module AutomaticHallucinationToleranceLevel
|
|
114
|
+
extend Deeprails::Internal::Type::Enum
|
|
115
|
+
|
|
116
|
+
LOW = :low
|
|
117
|
+
MEDIUM = :medium
|
|
118
|
+
HIGH = :high
|
|
119
|
+
|
|
120
|
+
# @!method self.values
|
|
121
|
+
# @return [Array<Symbol>]
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# The new action used to improve outputs that fail one or more guardrail metrics
|
|
125
|
+
# for the workflow events. May be `regen`, `fixit`, or `do_nothing`. ReGen runs
|
|
126
|
+
# the user's input prompt with minor induced variance. FixIt attempts to directly
|
|
127
|
+
# address the shortcomings of the output using the guardrail failure rationale. Do
|
|
128
|
+
# Nothing does not attempt any improvement.
|
|
129
|
+
module ImprovementAction
|
|
130
|
+
extend Deeprails::Internal::Type::Enum
|
|
131
|
+
|
|
132
|
+
REGEN = :regen
|
|
133
|
+
FIXIT = :fixit
|
|
134
|
+
DO_NOTHING = :do_nothing
|
|
135
|
+
|
|
136
|
+
# @!method self.values
|
|
137
|
+
# @return [Array<Symbol>]
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# New type of thresholds to use for the workflow, either `automatic` or `custom`.
|
|
141
|
+
# Automatic thresholds are assigned internally after the user specifies a
|
|
142
|
+
# qualitative tolerance for the metrics, whereas custom metrics allow the user to
|
|
143
|
+
# set the threshold for each metric as a floating point number between 0.0 and
|
|
144
|
+
# 1.0.
|
|
145
|
+
module ThresholdType
|
|
146
|
+
extend Deeprails::Internal::Type::Enum
|
|
147
|
+
|
|
148
|
+
AUTOMATIC = :automatic
|
|
149
|
+
CUSTOM = :custom
|
|
150
|
+
|
|
151
|
+
# @!method self.values
|
|
152
|
+
# @return [Array<Symbol>]
|
|
153
|
+
end
|
|
28
154
|
end
|
|
29
155
|
end
|
|
30
156
|
end
|
|
@@ -23,6 +23,17 @@ module Deeprails
|
|
|
23
23
|
# @return [String]
|
|
24
24
|
required :name, String
|
|
25
25
|
|
|
26
|
+
# @!attribute context_awareness
|
|
27
|
+
# Context includes any structured information that directly relates to the model’s
|
|
28
|
+
# input and expected output—e.g., the recent turn-by-turn history between an AI
|
|
29
|
+
# tutor and a student, facts or state passed through an agentic workflow, or other
|
|
30
|
+
# domain-specific signals your system already knows and wants the model to
|
|
31
|
+
# condition on. This field determines whether to enable context awareness for this
|
|
32
|
+
# monitor's evaluations. Defaults to false.
|
|
33
|
+
#
|
|
34
|
+
# @return [Boolean, nil]
|
|
35
|
+
optional :context_awareness, Deeprails::Internal::Type::Boolean
|
|
36
|
+
|
|
26
37
|
# @!attribute description
|
|
27
38
|
# Description of the new monitor.
|
|
28
39
|
#
|
|
@@ -42,7 +53,7 @@ module Deeprails
|
|
|
42
53
|
# @return [Boolean, nil]
|
|
43
54
|
optional :web_search, Deeprails::Internal::Type::Boolean
|
|
44
55
|
|
|
45
|
-
# @!method initialize(guardrail_metrics:, name:, description: nil, file_search: nil, web_search: nil, request_options: {})
|
|
56
|
+
# @!method initialize(guardrail_metrics:, name:, context_awareness: nil, description: nil, file_search: nil, web_search: nil, request_options: {})
|
|
46
57
|
# Some parameter documentations has been truncated, see
|
|
47
58
|
# {Deeprails::Models::MonitorCreateParams} for more details.
|
|
48
59
|
#
|
|
@@ -50,6 +61,8 @@ module Deeprails
|
|
|
50
61
|
#
|
|
51
62
|
# @param name [String] Name of the new monitor.
|
|
52
63
|
#
|
|
64
|
+
# @param context_awareness [Boolean] Context includes any structured information that directly relates to the model’s
|
|
65
|
+
#
|
|
53
66
|
# @param description [String] Description of the new monitor.
|
|
54
67
|
#
|
|
55
68
|
# @param file_search [Array<String>] An array of file IDs to search in the monitor's evaluations. Files must be uploa
|
|
@@ -5,7 +5,8 @@ module Deeprails
|
|
|
5
5
|
# @see Deeprails::Resources::Monitor#retrieve
|
|
6
6
|
class MonitorDetailResponse < Deeprails::Internal::Type::BaseModel
|
|
7
7
|
# @!attribute capabilities
|
|
8
|
-
# An array of capabilities associated with this monitor.
|
|
8
|
+
# An array of extended AI capabilities associated with this monitor. Can be
|
|
9
|
+
# `web_search`, `file_search`, and/or `context_awareness`.
|
|
9
10
|
#
|
|
10
11
|
# @return [Array<Deeprails::Models::MonitorDetailResponse::Capability>]
|
|
11
12
|
required :capabilities,
|
|
@@ -74,7 +75,7 @@ module Deeprails
|
|
|
74
75
|
# Some parameter documentations has been truncated, see
|
|
75
76
|
# {Deeprails::Models::MonitorDetailResponse} for more details.
|
|
76
77
|
#
|
|
77
|
-
# @param capabilities [Array<Deeprails::Models::MonitorDetailResponse::Capability>] An array of capabilities associated with this monitor.
|
|
78
|
+
# @param capabilities [Array<Deeprails::Models::MonitorDetailResponse::Capability>] An array of extended AI capabilities associated with this monitor. Can be `web\_
|
|
78
79
|
#
|
|
79
80
|
# @param created_at [Time] The time the monitor was created in UTC.
|
|
80
81
|
#
|
|
@@ -114,8 +115,9 @@ module Deeprails
|
|
|
114
115
|
|
|
115
116
|
# @!attribute model_input
|
|
116
117
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
117
|
-
# contain
|
|
118
|
-
#
|
|
118
|
+
# contain a `user_prompt` field. For ground_truth_adherence guardrail metric,
|
|
119
|
+
# `ground_truth` should be provided. When `context_awareness` is enabled,
|
|
120
|
+
# `context` should be provided.
|
|
119
121
|
#
|
|
120
122
|
# @return [Deeprails::Models::MonitorDetailResponse::Evaluation::ModelInput]
|
|
121
123
|
required :model_input, -> { Deeprails::MonitorDetailResponse::Evaluation::ModelInput }
|
|
@@ -223,6 +225,23 @@ module Deeprails
|
|
|
223
225
|
|
|
224
226
|
# @see Deeprails::Models::MonitorDetailResponse::Evaluation#model_input
|
|
225
227
|
class ModelInput < Deeprails::Internal::Type::BaseModel
|
|
228
|
+
# @!attribute user_prompt
|
|
229
|
+
# The user prompt used to generate the output.
|
|
230
|
+
#
|
|
231
|
+
# @return [String]
|
|
232
|
+
required :user_prompt, String
|
|
233
|
+
|
|
234
|
+
# @!attribute context
|
|
235
|
+
# Any structured information that directly relates to the model’s input and
|
|
236
|
+
# expected output—e.g., the recent turn-by-turn history between an AI tutor and a
|
|
237
|
+
# student, facts or state passed through an agentic workflow, or other
|
|
238
|
+
# domain-specific signals your system already knows and wants the model to
|
|
239
|
+
# condition on.
|
|
240
|
+
#
|
|
241
|
+
# @return [Array<Deeprails::Models::MonitorDetailResponse::Evaluation::ModelInput::Context>, nil]
|
|
242
|
+
optional :context,
|
|
243
|
+
-> { Deeprails::Internal::Type::ArrayOf[Deeprails::MonitorDetailResponse::Evaluation::ModelInput::Context] }
|
|
244
|
+
|
|
226
245
|
# @!attribute ground_truth
|
|
227
246
|
# The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
228
247
|
#
|
|
@@ -235,22 +254,42 @@ module Deeprails
|
|
|
235
254
|
# @return [String, nil]
|
|
236
255
|
optional :system_prompt, String
|
|
237
256
|
|
|
238
|
-
# @!
|
|
239
|
-
#
|
|
257
|
+
# @!method initialize(user_prompt:, context: nil, ground_truth: nil, system_prompt: nil)
|
|
258
|
+
# Some parameter documentations has been truncated, see
|
|
259
|
+
# {Deeprails::Models::MonitorDetailResponse::Evaluation::ModelInput} for more
|
|
260
|
+
# details.
|
|
240
261
|
#
|
|
241
|
-
# @return [String, nil]
|
|
242
|
-
optional :user_prompt, String
|
|
243
|
-
|
|
244
|
-
# @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
|
|
245
262
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
246
|
-
# contain
|
|
247
|
-
#
|
|
263
|
+
# contain a `user_prompt` field. For ground_truth_adherence guardrail metric,
|
|
264
|
+
# `ground_truth` should be provided. When `context_awareness` is enabled,
|
|
265
|
+
# `context` should be provided.
|
|
266
|
+
#
|
|
267
|
+
# @param user_prompt [String] The user prompt used to generate the output.
|
|
268
|
+
#
|
|
269
|
+
# @param context [Array<Deeprails::Models::MonitorDetailResponse::Evaluation::ModelInput::Context>] Any structured information that directly relates to the model’s input and expect
|
|
248
270
|
#
|
|
249
271
|
# @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
250
272
|
#
|
|
251
273
|
# @param system_prompt [String] The system prompt used to generate the output.
|
|
252
|
-
|
|
253
|
-
|
|
274
|
+
|
|
275
|
+
class Context < Deeprails::Internal::Type::BaseModel
|
|
276
|
+
# @!attribute content
|
|
277
|
+
# The content of the message.
|
|
278
|
+
#
|
|
279
|
+
# @return [String, nil]
|
|
280
|
+
optional :content, String
|
|
281
|
+
|
|
282
|
+
# @!attribute role
|
|
283
|
+
# The role of the speaker.
|
|
284
|
+
#
|
|
285
|
+
# @return [String, nil]
|
|
286
|
+
optional :role, String
|
|
287
|
+
|
|
288
|
+
# @!method initialize(content: nil, role: nil)
|
|
289
|
+
# @param content [String] The content of the message.
|
|
290
|
+
#
|
|
291
|
+
# @param role [String] The role of the speaker.
|
|
292
|
+
end
|
|
254
293
|
end
|
|
255
294
|
|
|
256
295
|
# Run mode for the evaluation. The run mode allows the user to optimize for speed,
|
|
@@ -5,7 +5,8 @@ module Deeprails
|
|
|
5
5
|
# @see Deeprails::Resources::Monitor#retrieve_event
|
|
6
6
|
class MonitorEventDetailResponse < Deeprails::Internal::Type::BaseModel
|
|
7
7
|
# @!attribute capabilities
|
|
8
|
-
# The capabilities associated with the monitor event.
|
|
8
|
+
# The extended AI capabilities associated with the monitor event. Can be
|
|
9
|
+
# `web_search`, `file_search`, and/or `context_awareness`.
|
|
9
10
|
#
|
|
10
11
|
# @return [Array<Deeprails::Models::MonitorEventDetailResponse::Capability>, nil]
|
|
11
12
|
optional :capabilities,
|
|
@@ -84,7 +85,10 @@ module Deeprails
|
|
|
84
85
|
optional :timestamp, Time
|
|
85
86
|
|
|
86
87
|
# @!method initialize(capabilities: nil, eval_time: nil, evaluation_result: nil, event_id: nil, files: nil, guardrail_metrics: nil, model_input: nil, model_output: nil, monitor_id: nil, nametag: nil, run_mode: nil, status: nil, timestamp: nil)
|
|
87
|
-
#
|
|
88
|
+
# Some parameter documentations has been truncated, see
|
|
89
|
+
# {Deeprails::Models::MonitorEventDetailResponse} for more details.
|
|
90
|
+
#
|
|
91
|
+
# @param capabilities [Array<Deeprails::Models::MonitorEventDetailResponse::Capability>] The extended AI capabilities associated with the monitor event. Can be `web_sear
|
|
88
92
|
#
|
|
89
93
|
# @param eval_time [String] The time spent on the evaluation in seconds.
|
|
90
94
|
#
|
|
@@ -9,8 +9,8 @@ module Deeprails
|
|
|
9
9
|
|
|
10
10
|
# @!attribute model_input
|
|
11
11
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
12
|
-
# contain
|
|
13
|
-
#
|
|
12
|
+
# contain a `user_prompt` field. For ground_truth_adherence guardrail metric,
|
|
13
|
+
# `ground_truth` should be provided.
|
|
14
14
|
#
|
|
15
15
|
# @return [Deeprails::Models::MonitorSubmitEventParams::ModelInput]
|
|
16
16
|
required :model_input, -> { Deeprails::MonitorSubmitEventParams::ModelInput }
|
|
@@ -30,8 +30,8 @@ module Deeprails
|
|
|
30
30
|
# @!attribute run_mode
|
|
31
31
|
# Run mode for the monitor event. The run mode allows the user to optimize for
|
|
32
32
|
# speed, accuracy, and cost by determining which models are used to evaluate the
|
|
33
|
-
# event. Available run modes include `
|
|
34
|
-
# `economy`. Defaults to `smart`.
|
|
33
|
+
# event. Available run modes include `precision_plus_codex`, `precision_plus`,
|
|
34
|
+
# `precision`, `smart`, and `economy`. Defaults to `smart`.
|
|
35
35
|
#
|
|
36
36
|
# @return [Symbol, Deeprails::Models::MonitorSubmitEventParams::RunMode, nil]
|
|
37
37
|
optional :run_mode, enum: -> { Deeprails::MonitorSubmitEventParams::RunMode }
|
|
@@ -51,6 +51,23 @@ module Deeprails
|
|
|
51
51
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
52
52
|
|
|
53
53
|
class ModelInput < Deeprails::Internal::Type::BaseModel
|
|
54
|
+
# @!attribute user_prompt
|
|
55
|
+
# The user prompt used to generate the output.
|
|
56
|
+
#
|
|
57
|
+
# @return [String]
|
|
58
|
+
required :user_prompt, String
|
|
59
|
+
|
|
60
|
+
# @!attribute context
|
|
61
|
+
# Any structured information that directly relates to the model’s input and
|
|
62
|
+
# expected output—e.g., the recent turn-by-turn history between an AI tutor and a
|
|
63
|
+
# student, facts or state passed through an agentic workflow, or other
|
|
64
|
+
# domain-specific signals your system already knows and wants the model to
|
|
65
|
+
# condition on.
|
|
66
|
+
#
|
|
67
|
+
# @return [Array<Deeprails::Models::MonitorSubmitEventParams::ModelInput::Context>, nil]
|
|
68
|
+
optional :context,
|
|
69
|
+
-> { Deeprails::Internal::Type::ArrayOf[Deeprails::MonitorSubmitEventParams::ModelInput::Context] }
|
|
70
|
+
|
|
54
71
|
# @!attribute ground_truth
|
|
55
72
|
# The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
56
73
|
#
|
|
@@ -63,31 +80,50 @@ module Deeprails
|
|
|
63
80
|
# @return [String, nil]
|
|
64
81
|
optional :system_prompt, String
|
|
65
82
|
|
|
66
|
-
# @!
|
|
67
|
-
#
|
|
83
|
+
# @!method initialize(user_prompt:, context: nil, ground_truth: nil, system_prompt: nil)
|
|
84
|
+
# Some parameter documentations has been truncated, see
|
|
85
|
+
# {Deeprails::Models::MonitorSubmitEventParams::ModelInput} for more details.
|
|
68
86
|
#
|
|
69
|
-
# @return [String, nil]
|
|
70
|
-
optional :user_prompt, String
|
|
71
|
-
|
|
72
|
-
# @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
|
|
73
87
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
74
|
-
# contain
|
|
75
|
-
#
|
|
88
|
+
# contain a `user_prompt` field. For ground_truth_adherence guardrail metric,
|
|
89
|
+
# `ground_truth` should be provided.
|
|
90
|
+
#
|
|
91
|
+
# @param user_prompt [String] The user prompt used to generate the output.
|
|
92
|
+
#
|
|
93
|
+
# @param context [Array<Deeprails::Models::MonitorSubmitEventParams::ModelInput::Context>] Any structured information that directly relates to the model’s input and expect
|
|
76
94
|
#
|
|
77
95
|
# @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
78
96
|
#
|
|
79
97
|
# @param system_prompt [String] The system prompt used to generate the output.
|
|
80
|
-
|
|
81
|
-
|
|
98
|
+
|
|
99
|
+
class Context < Deeprails::Internal::Type::BaseModel
|
|
100
|
+
# @!attribute content
|
|
101
|
+
# The content of the message.
|
|
102
|
+
#
|
|
103
|
+
# @return [String, nil]
|
|
104
|
+
optional :content, String
|
|
105
|
+
|
|
106
|
+
# @!attribute role
|
|
107
|
+
# The role of the speaker.
|
|
108
|
+
#
|
|
109
|
+
# @return [String, nil]
|
|
110
|
+
optional :role, String
|
|
111
|
+
|
|
112
|
+
# @!method initialize(content: nil, role: nil)
|
|
113
|
+
# @param content [String] The content of the message.
|
|
114
|
+
#
|
|
115
|
+
# @param role [String] The role of the speaker.
|
|
116
|
+
end
|
|
82
117
|
end
|
|
83
118
|
|
|
84
119
|
# Run mode for the monitor event. The run mode allows the user to optimize for
|
|
85
120
|
# speed, accuracy, and cost by determining which models are used to evaluate the
|
|
86
|
-
# event. Available run modes include `
|
|
87
|
-
# `economy`. Defaults to `smart`.
|
|
121
|
+
# event. Available run modes include `precision_plus_codex`, `precision_plus`,
|
|
122
|
+
# `precision`, `smart`, and `economy`. Defaults to `smart`.
|
|
88
123
|
module RunMode
|
|
89
124
|
extend Deeprails::Internal::Type::Enum
|
|
90
125
|
|
|
126
|
+
PRECISION_PLUS_CODEX = :precision_plus_codex
|
|
91
127
|
PRECISION_PLUS = :precision_plus
|
|
92
128
|
PRECISION = :precision
|
|
93
129
|
SMART = :smart
|
|
@@ -8,13 +8,28 @@ module Deeprails
|
|
|
8
8
|
include Deeprails::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
10
|
# @!attribute description
|
|
11
|
-
#
|
|
11
|
+
# New description of the monitor.
|
|
12
12
|
#
|
|
13
13
|
# @return [String, nil]
|
|
14
14
|
optional :description, String
|
|
15
15
|
|
|
16
|
+
# @!attribute file_search
|
|
17
|
+
# An array of file IDs to search in the monitor's evaluations. Files must be
|
|
18
|
+
# uploaded via the DeepRails API first.
|
|
19
|
+
#
|
|
20
|
+
# @return [Array<String>, nil]
|
|
21
|
+
optional :file_search, Deeprails::Internal::Type::ArrayOf[String]
|
|
22
|
+
|
|
23
|
+
# @!attribute guardrail_metrics
|
|
24
|
+
# An array of the new guardrail metrics that model input and output pairs will be
|
|
25
|
+
# evaluated on.
|
|
26
|
+
#
|
|
27
|
+
# @return [Array<Symbol, Deeprails::Models::MonitorUpdateParams::GuardrailMetric>, nil]
|
|
28
|
+
optional :guardrail_metrics,
|
|
29
|
+
-> { Deeprails::Internal::Type::ArrayOf[enum: Deeprails::MonitorUpdateParams::GuardrailMetric] }
|
|
30
|
+
|
|
16
31
|
# @!attribute name
|
|
17
|
-
#
|
|
32
|
+
# New name of the monitor.
|
|
18
33
|
#
|
|
19
34
|
# @return [String, nil]
|
|
20
35
|
optional :name, String
|
|
@@ -26,18 +41,44 @@ module Deeprails
|
|
|
26
41
|
# @return [Symbol, Deeprails::Models::MonitorUpdateParams::Status, nil]
|
|
27
42
|
optional :status, enum: -> { Deeprails::MonitorUpdateParams::Status }
|
|
28
43
|
|
|
29
|
-
# @!
|
|
44
|
+
# @!attribute web_search
|
|
45
|
+
# Whether to enable web search for this monitor's evaluations.
|
|
46
|
+
#
|
|
47
|
+
# @return [Boolean, nil]
|
|
48
|
+
optional :web_search, Deeprails::Internal::Type::Boolean
|
|
49
|
+
|
|
50
|
+
# @!method initialize(description: nil, file_search: nil, guardrail_metrics: nil, name: nil, status: nil, web_search: nil, request_options: {})
|
|
30
51
|
# Some parameter documentations has been truncated, see
|
|
31
52
|
# {Deeprails::Models::MonitorUpdateParams} for more details.
|
|
32
53
|
#
|
|
33
|
-
# @param description [String]
|
|
54
|
+
# @param description [String] New description of the monitor.
|
|
55
|
+
#
|
|
56
|
+
# @param file_search [Array<String>] An array of file IDs to search in the monitor's evaluations. Files must be uploa
|
|
57
|
+
#
|
|
58
|
+
# @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorUpdateParams::GuardrailMetric>] An array of the new guardrail metrics that model input and output pairs will be
|
|
34
59
|
#
|
|
35
|
-
# @param name [String]
|
|
60
|
+
# @param name [String] New name of the monitor.
|
|
36
61
|
#
|
|
37
62
|
# @param status [Symbol, Deeprails::Models::MonitorUpdateParams::Status] Status of the monitor. Can be `active` or `inactive`. Inactive monitors no lon
|
|
38
63
|
#
|
|
64
|
+
# @param web_search [Boolean] Whether to enable web search for this monitor's evaluations.
|
|
65
|
+
#
|
|
39
66
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
40
67
|
|
|
68
|
+
module GuardrailMetric
|
|
69
|
+
extend Deeprails::Internal::Type::Enum
|
|
70
|
+
|
|
71
|
+
CORRECTNESS = :correctness
|
|
72
|
+
COMPLETENESS = :completeness
|
|
73
|
+
INSTRUCTION_ADHERENCE = :instruction_adherence
|
|
74
|
+
CONTEXT_ADHERENCE = :context_adherence
|
|
75
|
+
GROUND_TRUTH_ADHERENCE = :ground_truth_adherence
|
|
76
|
+
COMPREHENSIVE_SAFETY = :comprehensive_safety
|
|
77
|
+
|
|
78
|
+
# @!method self.values
|
|
79
|
+
# @return [Array<Symbol>]
|
|
80
|
+
end
|
|
81
|
+
|
|
41
82
|
# Status of the monitor. Can be `active` or `inactive`. Inactive monitors no
|
|
42
83
|
# longer record and evaluate events.
|
|
43
84
|
module Status
|