deeprails 0.18.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +62 -0
  3. data/README.md +1 -1
  4. data/lib/deeprails/internal/stream.rb +29 -0
  5. data/lib/deeprails/internal/transport/base_client.rb +3 -1
  6. data/lib/deeprails/internal/transport/pooled_net_requester.rb +12 -10
  7. data/lib/deeprails/internal/type/base_stream.rb +83 -0
  8. data/lib/deeprails/internal/util.rb +7 -2
  9. data/lib/deeprails/models/defend_create_workflow_params.rb +14 -1
  10. data/lib/deeprails/models/defend_response.rb +134 -10
  11. data/lib/deeprails/models/defend_submit_and_stream_event_params.rb +83 -0
  12. data/lib/deeprails/models/defend_submit_and_stream_event_response.rb +7 -0
  13. data/lib/deeprails/models/defend_submit_event_params.rb +52 -16
  14. data/lib/deeprails/models/defend_update_response.rb +9 -1
  15. data/lib/deeprails/models/defend_update_workflow_params.rb +131 -5
  16. data/lib/deeprails/models/monitor_create_params.rb +14 -1
  17. data/lib/deeprails/models/monitor_detail_response.rb +53 -14
  18. data/lib/deeprails/models/monitor_event_detail_response.rb +6 -2
  19. data/lib/deeprails/models/monitor_submit_event_params.rb +52 -16
  20. data/lib/deeprails/models/monitor_update_params.rb +46 -5
  21. data/lib/deeprails/models/workflow_event_detail_response.rb +88 -16
  22. data/lib/deeprails/models/workflow_event_response.rb +9 -1
  23. data/lib/deeprails/models.rb +2 -0
  24. data/lib/deeprails/resources/defend.rb +67 -4
  25. data/lib/deeprails/resources/monitor.rb +14 -6
  26. data/lib/deeprails/version.rb +1 -1
  27. data/lib/deeprails.rb +5 -0
  28. data/manifest.yaml +1 -0
  29. data/rbi/deeprails/internal/stream.rbi +20 -0
  30. data/rbi/deeprails/internal/transport/base_client.rbi +18 -2
  31. data/rbi/deeprails/internal/type/base_stream.rbi +75 -0
  32. data/rbi/deeprails/models/defend_create_workflow_params.rbi +21 -0
  33. data/rbi/deeprails/models/defend_response.rbi +286 -14
  34. data/rbi/deeprails/models/defend_submit_and_stream_event_params.rbi +148 -0
  35. data/rbi/deeprails/models/defend_submit_and_stream_event_response.rbi +7 -0
  36. data/rbi/deeprails/models/defend_submit_event_params.rbi +108 -25
  37. data/rbi/deeprails/models/defend_update_response.rbi +14 -3
  38. data/rbi/deeprails/models/defend_update_workflow_params.rbi +298 -4
  39. data/rbi/deeprails/models/monitor_create_params.rbi +21 -0
  40. data/rbi/deeprails/models/monitor_detail_response.rbi +103 -22
  41. data/rbi/deeprails/models/monitor_event_detail_response.rbi +4 -2
  42. data/rbi/deeprails/models/monitor_submit_event_params.rbi +108 -25
  43. data/rbi/deeprails/models/monitor_update_params.rbi +107 -4
  44. data/rbi/deeprails/models/workflow_event_detail_response.rbi +170 -24
  45. data/rbi/deeprails/models/workflow_event_response.rbi +8 -0
  46. data/rbi/deeprails/models.rbi +3 -0
  47. data/rbi/deeprails/resources/defend.rbi +96 -6
  48. data/rbi/deeprails/resources/monitor.rbi +28 -8
  49. data/sig/deeprails/internal/stream.rbs +9 -0
  50. data/sig/deeprails/internal/type/base_stream.rbs +38 -0
  51. data/sig/deeprails/models/defend_create_workflow_params.rbs +7 -0
  52. data/sig/deeprails/models/defend_response.rbs +114 -8
  53. data/sig/deeprails/models/defend_submit_and_stream_event_params.rbs +74 -0
  54. data/sig/deeprails/models/defend_submit_and_stream_event_response.rbs +5 -0
  55. data/sig/deeprails/models/defend_submit_event_params.rbs +39 -10
  56. data/sig/deeprails/models/defend_update_response.rbs +10 -3
  57. data/sig/deeprails/models/defend_update_workflow_params.rbs +103 -1
  58. data/sig/deeprails/models/monitor_create_params.rbs +7 -0
  59. data/sig/deeprails/models/monitor_detail_response.rbs +36 -9
  60. data/sig/deeprails/models/monitor_submit_event_params.rbs +39 -10
  61. data/sig/deeprails/models/monitor_update_params.rbs +45 -1
  62. data/sig/deeprails/models/workflow_event_detail_response.rbs +88 -15
  63. data/sig/deeprails/models/workflow_event_response.rbs +5 -0
  64. data/sig/deeprails/models.rbs +2 -0
  65. data/sig/deeprails/resources/defend.rbs +20 -0
  66. data/sig/deeprails/resources/monitor.rbs +4 -0
  67. metadata +28 -2
@@ -9,8 +9,8 @@ module Deeprails
9
9
 
10
10
  # @!attribute model_input
11
11
  # A dictionary of inputs sent to the LLM to generate output. The dictionary must
12
- # contain at least a `user_prompt` field or a `system_prompt` field. For the
13
- # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
12
+ # contain a `user_prompt` field. For the ground_truth_adherence guardrail metric,
13
+ # `ground_truth` should be provided.
14
14
  #
15
15
  # @return [Deeprails::Models::DefendSubmitEventParams::ModelInput]
16
16
  required :model_input, -> { Deeprails::DefendSubmitEventParams::ModelInput }
@@ -30,8 +30,8 @@ module Deeprails
30
30
  # @!attribute run_mode
31
31
  # Run mode for the workflow event. The run mode allows the user to optimize for
32
32
  # speed, accuracy, and cost by determining which models are used to evaluate the
33
- # event. Available run modes include `precision_plus`, `precision`, `smart`, and
34
- # `economy`. Defaults to `smart`.
33
+ # event. Available run modes include `precision_plus_codex`, `precision_plus`,
34
+ # `precision`, `smart`, and `economy`. Defaults to `smart`.
35
35
  #
36
36
  # @return [Symbol, Deeprails::Models::DefendSubmitEventParams::RunMode]
37
37
  required :run_mode, enum: -> { Deeprails::DefendSubmitEventParams::RunMode }
@@ -59,6 +59,23 @@ module Deeprails
59
59
  # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
60
60
 
61
61
  class ModelInput < Deeprails::Internal::Type::BaseModel
62
+ # @!attribute user_prompt
63
+ # The user prompt used to generate the output.
64
+ #
65
+ # @return [String]
66
+ required :user_prompt, String
67
+
68
+ # @!attribute context
69
+ # Any structured information that directly relates to the model’s input and
70
+ # expected output—e.g., the recent turn-by-turn history between an AI tutor and a
71
+ # student, facts or state passed through an agentic workflow, or other
72
+ # domain-specific signals your system already knows and wants the model to
73
+ # condition on.
74
+ #
75
+ # @return [Array<Deeprails::Models::DefendSubmitEventParams::ModelInput::Context>, nil]
76
+ optional :context,
77
+ -> { Deeprails::Internal::Type::ArrayOf[Deeprails::DefendSubmitEventParams::ModelInput::Context] }
78
+
62
79
  # @!attribute ground_truth
63
80
  # The ground truth for evaluating the Ground Truth Adherence guardrail.
64
81
  #
@@ -71,31 +88,50 @@ module Deeprails
71
88
  # @return [String, nil]
72
89
  optional :system_prompt, String
73
90
 
74
- # @!attribute user_prompt
75
- # The user prompt used to generate the output.
91
+ # @!method initialize(user_prompt:, context: nil, ground_truth: nil, system_prompt: nil)
92
+ # Some parameter documentations has been truncated, see
93
+ # {Deeprails::Models::DefendSubmitEventParams::ModelInput} for more details.
76
94
  #
77
- # @return [String, nil]
78
- optional :user_prompt, String
79
-
80
- # @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
81
95
  # A dictionary of inputs sent to the LLM to generate output. The dictionary must
82
- # contain at least a `user_prompt` field or a `system_prompt` field. For the
83
- # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
96
+ # contain a `user_prompt` field. For the ground_truth_adherence guardrail metric,
97
+ # `ground_truth` should be provided.
98
+ #
99
+ # @param user_prompt [String] The user prompt used to generate the output.
100
+ #
101
+ # @param context [Array<Deeprails::Models::DefendSubmitEventParams::ModelInput::Context>] Any structured information that directly relates to the model’s input and expect
84
102
  #
85
103
  # @param ground_truth [String] The ground truth for evaluating the Ground Truth Adherence guardrail.
86
104
  #
87
105
  # @param system_prompt [String] The system prompt used to generate the output.
88
- #
89
- # @param user_prompt [String] The user prompt used to generate the output.
106
+
107
+ class Context < Deeprails::Internal::Type::BaseModel
108
+ # @!attribute content
109
+ # The content of the message.
110
+ #
111
+ # @return [String, nil]
112
+ optional :content, String
113
+
114
+ # @!attribute role
115
+ # The role of the speaker.
116
+ #
117
+ # @return [String, nil]
118
+ optional :role, String
119
+
120
+ # @!method initialize(content: nil, role: nil)
121
+ # @param content [String] The content of the message.
122
+ #
123
+ # @param role [String] The role of the speaker.
124
+ end
90
125
  end
91
126
 
92
127
  # Run mode for the workflow event. The run mode allows the user to optimize for
93
128
  # speed, accuracy, and cost by determining which models are used to evaluate the
94
- # event. Available run modes include `precision_plus`, `precision`, `smart`, and
95
- # `economy`. Defaults to `smart`.
129
+ # event. Available run modes include `precision_plus_codex`, `precision_plus`,
130
+ # `precision`, `smart`, and `economy`. Defaults to `smart`.
96
131
  module RunMode
97
132
  extend Deeprails::Internal::Type::Enum
98
133
 
134
+ PRECISION_PLUS_CODEX = :precision_plus_codex
99
135
  PRECISION_PLUS = :precision_plus
100
136
  PRECISION = :precision
101
137
  SMART = :smart
@@ -23,7 +23,13 @@ module Deeprails
23
23
  # @return [String]
24
24
  required :workflow_id, String
25
25
 
26
- # @!method initialize(modified_at:, status:, workflow_id:)
26
+ # @!attribute name
27
+ # The name of the workflow.
28
+ #
29
+ # @return [String, nil]
30
+ optional :name, String
31
+
32
+ # @!method initialize(modified_at:, status:, workflow_id:, name: nil)
27
33
  # Some parameter documentations has been truncated, see
28
34
  # {Deeprails::Models::DefendUpdateResponse} for more details.
29
35
  #
@@ -32,6 +38,8 @@ module Deeprails
32
38
  # @param status [Symbol, Deeprails::Models::DefendUpdateResponse::Status] Status of the selected workflow. May be `inactive` or `active`. Inactive workf
33
39
  #
34
40
  # @param workflow_id [String] A unique workflow ID.
41
+ #
42
+ # @param name [String] The name of the workflow.
35
43
 
36
44
  # Status of the selected workflow. May be `inactive` or `active`. Inactive
37
45
  # workflows will not accept events.
@@ -7,24 +7,150 @@ module Deeprails
7
7
  extend Deeprails::Internal::Type::RequestParameters::Converter
8
8
  include Deeprails::Internal::Type::RequestParameters
9
9
 
10
+ # @!attribute automatic_hallucination_tolerance_levels
11
+ # New mapping of guardrail metrics to hallucination tolerance levels (either
12
+ # `low`, `medium`, or `high`) to be used when `threshold_type` is set to
13
+ # `automatic`. Possible metrics are `completeness`, `instruction_adherence`,
14
+ # `context_adherence`, `ground_truth_adherence`, or `comprehensive_safety`.
15
+ #
16
+ # @return [Hash{Symbol=>Symbol, Deeprails::Models::DefendUpdateWorkflowParams::AutomaticHallucinationToleranceLevel}, nil]
17
+ optional :automatic_hallucination_tolerance_levels,
18
+ -> { Deeprails::Internal::Type::HashOf[enum: Deeprails::DefendUpdateWorkflowParams::AutomaticHallucinationToleranceLevel] }
19
+
20
+ # @!attribute context_awareness
21
+ # Whether to enable context awareness for this workflow's evaluations.
22
+ #
23
+ # @return [Boolean, nil]
24
+ optional :context_awareness, Deeprails::Internal::Type::Boolean
25
+
26
+ # @!attribute custom_hallucination_threshold_values
27
+ # New mapping of guardrail metrics to floating point threshold values to be used
28
+ # when `threshold_type` is set to `custom`. Possible metrics are `correctness`,
29
+ # `completeness`, `instruction_adherence`, `context_adherence`,
30
+ # `ground_truth_adherence`, or `comprehensive_safety`.
31
+ #
32
+ # @return [Hash{Symbol=>Float}, nil]
33
+ optional :custom_hallucination_threshold_values, Deeprails::Internal::Type::HashOf[Float]
34
+
10
35
  # @!attribute description
11
- # Description for the workflow.
36
+ # New description for the workflow.
12
37
  #
13
38
  # @return [String, nil]
14
39
  optional :description, String
15
40
 
41
+ # @!attribute file_search
42
+ # An array of file IDs to search in the workflow's evaluations. Files must be
43
+ # uploaded via the DeepRails API first.
44
+ #
45
+ # @return [Array<String>, nil]
46
+ optional :file_search, Deeprails::Internal::Type::ArrayOf[String]
47
+
48
+ # @!attribute improvement_action
49
+ # The new action used to improve outputs that fail one or more guardrail metrics
50
+ # for the workflow events. May be `regen`, `fixit`, or `do_nothing`. ReGen runs
51
+ # the user's input prompt with minor induced variance. FixIt attempts to directly
52
+ # address the shortcomings of the output using the guardrail failure rationale. Do
53
+ # Nothing does not attempt any improvement.
54
+ #
55
+ # @return [Symbol, Deeprails::Models::DefendUpdateWorkflowParams::ImprovementAction, nil]
56
+ optional :improvement_action, enum: -> { Deeprails::DefendUpdateWorkflowParams::ImprovementAction }
57
+
58
+ # @!attribute max_improvement_attempts
59
+ # Max. number of improvement action attempts until a given event passes the
60
+ # guardrails. Defaults to 10.
61
+ #
62
+ # @return [Integer, nil]
63
+ optional :max_improvement_attempts, Integer
64
+
16
65
  # @!attribute name
17
- # Name of the workflow.
66
+ # New name for the workflow.
18
67
  #
19
68
  # @return [String, nil]
20
69
  optional :name, String
21
70
 
22
- # @!method initialize(description: nil, name: nil, request_options: {})
23
- # @param description [String] Description for the workflow.
71
+ # @!attribute threshold_type
72
+ # New type of thresholds to use for the workflow, either `automatic` or `custom`.
73
+ # Automatic thresholds are assigned internally after the user specifies a
74
+ # qualitative tolerance for the metrics, whereas custom metrics allow the user to
75
+ # set the threshold for each metric as a floating point number between 0.0 and
76
+ # 1.0.
77
+ #
78
+ # @return [Symbol, Deeprails::Models::DefendUpdateWorkflowParams::ThresholdType, nil]
79
+ optional :threshold_type, enum: -> { Deeprails::DefendUpdateWorkflowParams::ThresholdType }
80
+
81
+ # @!attribute web_search
82
+ # Whether to enable web search for this workflow's evaluations.
83
+ #
84
+ # @return [Boolean, nil]
85
+ optional :web_search, Deeprails::Internal::Type::Boolean
86
+
87
+ # @!method initialize(automatic_hallucination_tolerance_levels: nil, context_awareness: nil, custom_hallucination_threshold_values: nil, description: nil, file_search: nil, improvement_action: nil, max_improvement_attempts: nil, name: nil, threshold_type: nil, web_search: nil, request_options: {})
88
+ # Some parameter documentations has been truncated, see
89
+ # {Deeprails::Models::DefendUpdateWorkflowParams} for more details.
90
+ #
91
+ # @param automatic_hallucination_tolerance_levels [Hash{Symbol=>Symbol, Deeprails::Models::DefendUpdateWorkflowParams::AutomaticHallucinationToleranceLevel}] New mapping of guardrail metrics to hallucination tolerance levels
24
92
  #
25
- # @param name [String] Name of the workflow.
93
+ # @param context_awareness [Boolean] Whether to enable context awareness for this workflow's evaluations.
94
+ #
95
+ # @param custom_hallucination_threshold_values [Hash{Symbol=>Float}] New mapping of guardrail metrics to floating point threshold values to be used w
96
+ #
97
+ # @param description [String] New description for the workflow.
98
+ #
99
+ # @param file_search [Array<String>] An array of file IDs to search in the workflow's evaluations. Files must be uplo
100
+ #
101
+ # @param improvement_action [Symbol, Deeprails::Models::DefendUpdateWorkflowParams::ImprovementAction] The new action used to improve outputs that fail one or more guardrail metrics f
102
+ #
103
+ # @param max_improvement_attempts [Integer] Max. number of improvement action attempts until a given event passes the guardr
104
+ #
105
+ # @param name [String] New name for the workflow.
106
+ #
107
+ # @param threshold_type [Symbol, Deeprails::Models::DefendUpdateWorkflowParams::ThresholdType] New type of thresholds to use for the workflow, either `automatic` or `custom`.
108
+ #
109
+ # @param web_search [Boolean] Whether to enable web search for this workflow's evaluations.
26
110
  #
27
111
  # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
112
+
113
+ module AutomaticHallucinationToleranceLevel
114
+ extend Deeprails::Internal::Type::Enum
115
+
116
+ LOW = :low
117
+ MEDIUM = :medium
118
+ HIGH = :high
119
+
120
+ # @!method self.values
121
+ # @return [Array<Symbol>]
122
+ end
123
+
124
+ # The new action used to improve outputs that fail one or more guardrail metrics
125
+ # for the workflow events. May be `regen`, `fixit`, or `do_nothing`. ReGen runs
126
+ # the user's input prompt with minor induced variance. FixIt attempts to directly
127
+ # address the shortcomings of the output using the guardrail failure rationale. Do
128
+ # Nothing does not attempt any improvement.
129
+ module ImprovementAction
130
+ extend Deeprails::Internal::Type::Enum
131
+
132
+ REGEN = :regen
133
+ FIXIT = :fixit
134
+ DO_NOTHING = :do_nothing
135
+
136
+ # @!method self.values
137
+ # @return [Array<Symbol>]
138
+ end
139
+
140
+ # New type of thresholds to use for the workflow, either `automatic` or `custom`.
141
+ # Automatic thresholds are assigned internally after the user specifies a
142
+ # qualitative tolerance for the metrics, whereas custom metrics allow the user to
143
+ # set the threshold for each metric as a floating point number between 0.0 and
144
+ # 1.0.
145
+ module ThresholdType
146
+ extend Deeprails::Internal::Type::Enum
147
+
148
+ AUTOMATIC = :automatic
149
+ CUSTOM = :custom
150
+
151
+ # @!method self.values
152
+ # @return [Array<Symbol>]
153
+ end
28
154
  end
29
155
  end
30
156
  end
@@ -23,6 +23,17 @@ module Deeprails
23
23
  # @return [String]
24
24
  required :name, String
25
25
 
26
+ # @!attribute context_awareness
27
+ # Context includes any structured information that directly relates to the model’s
28
+ # input and expected output—e.g., the recent turn-by-turn history between an AI
29
+ # tutor and a student, facts or state passed through an agentic workflow, or other
30
+ # domain-specific signals your system already knows and wants the model to
31
+ # condition on. This field determines whether to enable context awareness for this
32
+ # monitor's evaluations. Defaults to false.
33
+ #
34
+ # @return [Boolean, nil]
35
+ optional :context_awareness, Deeprails::Internal::Type::Boolean
36
+
26
37
  # @!attribute description
27
38
  # Description of the new monitor.
28
39
  #
@@ -42,7 +53,7 @@ module Deeprails
42
53
  # @return [Boolean, nil]
43
54
  optional :web_search, Deeprails::Internal::Type::Boolean
44
55
 
45
- # @!method initialize(guardrail_metrics:, name:, description: nil, file_search: nil, web_search: nil, request_options: {})
56
+ # @!method initialize(guardrail_metrics:, name:, context_awareness: nil, description: nil, file_search: nil, web_search: nil, request_options: {})
46
57
  # Some parameter documentations has been truncated, see
47
58
  # {Deeprails::Models::MonitorCreateParams} for more details.
48
59
  #
@@ -50,6 +61,8 @@ module Deeprails
50
61
  #
51
62
  # @param name [String] Name of the new monitor.
52
63
  #
64
+ # @param context_awareness [Boolean] Context includes any structured information that directly relates to the model’s
65
+ #
53
66
  # @param description [String] Description of the new monitor.
54
67
  #
55
68
  # @param file_search [Array<String>] An array of file IDs to search in the monitor's evaluations. Files must be uploa
@@ -5,7 +5,8 @@ module Deeprails
5
5
  # @see Deeprails::Resources::Monitor#retrieve
6
6
  class MonitorDetailResponse < Deeprails::Internal::Type::BaseModel
7
7
  # @!attribute capabilities
8
- # An array of capabilities associated with this monitor.
8
+ # An array of extended AI capabilities associated with this monitor. Can be
9
+ # `web_search`, `file_search`, and/or `context_awareness`.
9
10
  #
10
11
  # @return [Array<Deeprails::Models::MonitorDetailResponse::Capability>]
11
12
  required :capabilities,
@@ -74,7 +75,7 @@ module Deeprails
74
75
  # Some parameter documentations has been truncated, see
75
76
  # {Deeprails::Models::MonitorDetailResponse} for more details.
76
77
  #
77
- # @param capabilities [Array<Deeprails::Models::MonitorDetailResponse::Capability>] An array of capabilities associated with this monitor.
78
+ # @param capabilities [Array<Deeprails::Models::MonitorDetailResponse::Capability>] An array of extended AI capabilities associated with this monitor. Can be `web\_
78
79
  #
79
80
  # @param created_at [Time] The time the monitor was created in UTC.
80
81
  #
@@ -114,8 +115,9 @@ module Deeprails
114
115
 
115
116
  # @!attribute model_input
116
117
  # A dictionary of inputs sent to the LLM to generate output. The dictionary must
117
- # contain at least a `user_prompt` field or a `system_prompt` field. For
118
- # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
118
+ # contain a `user_prompt` field. For ground_truth_adherence guardrail metric,
119
+ # `ground_truth` should be provided. When `context_awareness` is enabled,
120
+ # `context` should be provided.
119
121
  #
120
122
  # @return [Deeprails::Models::MonitorDetailResponse::Evaluation::ModelInput]
121
123
  required :model_input, -> { Deeprails::MonitorDetailResponse::Evaluation::ModelInput }
@@ -223,6 +225,23 @@ module Deeprails
223
225
 
224
226
  # @see Deeprails::Models::MonitorDetailResponse::Evaluation#model_input
225
227
  class ModelInput < Deeprails::Internal::Type::BaseModel
228
+ # @!attribute user_prompt
229
+ # The user prompt used to generate the output.
230
+ #
231
+ # @return [String]
232
+ required :user_prompt, String
233
+
234
+ # @!attribute context
235
+ # Any structured information that directly relates to the model’s input and
236
+ # expected output—e.g., the recent turn-by-turn history between an AI tutor and a
237
+ # student, facts or state passed through an agentic workflow, or other
238
+ # domain-specific signals your system already knows and wants the model to
239
+ # condition on.
240
+ #
241
+ # @return [Array<Deeprails::Models::MonitorDetailResponse::Evaluation::ModelInput::Context>, nil]
242
+ optional :context,
243
+ -> { Deeprails::Internal::Type::ArrayOf[Deeprails::MonitorDetailResponse::Evaluation::ModelInput::Context] }
244
+
226
245
  # @!attribute ground_truth
227
246
  # The ground truth for evaluating Ground Truth Adherence guardrail.
228
247
  #
@@ -235,22 +254,42 @@ module Deeprails
235
254
  # @return [String, nil]
236
255
  optional :system_prompt, String
237
256
 
238
- # @!attribute user_prompt
239
- # The user prompt used to generate the output.
257
+ # @!method initialize(user_prompt:, context: nil, ground_truth: nil, system_prompt: nil)
258
+ # Some parameter documentations has been truncated, see
259
+ # {Deeprails::Models::MonitorDetailResponse::Evaluation::ModelInput} for more
260
+ # details.
240
261
  #
241
- # @return [String, nil]
242
- optional :user_prompt, String
243
-
244
- # @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
245
262
  # A dictionary of inputs sent to the LLM to generate output. The dictionary must
246
- # contain at least a `user_prompt` field or a `system_prompt` field. For
247
- # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
263
+ # contain a `user_prompt` field. For ground_truth_adherence guardrail metric,
264
+ # `ground_truth` should be provided. When `context_awareness` is enabled,
265
+ # `context` should be provided.
266
+ #
267
+ # @param user_prompt [String] The user prompt used to generate the output.
268
+ #
269
+ # @param context [Array<Deeprails::Models::MonitorDetailResponse::Evaluation::ModelInput::Context>] Any structured information that directly relates to the model’s input and expect
248
270
  #
249
271
  # @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
250
272
  #
251
273
  # @param system_prompt [String] The system prompt used to generate the output.
252
- #
253
- # @param user_prompt [String] The user prompt used to generate the output.
274
+
275
+ class Context < Deeprails::Internal::Type::BaseModel
276
+ # @!attribute content
277
+ # The content of the message.
278
+ #
279
+ # @return [String, nil]
280
+ optional :content, String
281
+
282
+ # @!attribute role
283
+ # The role of the speaker.
284
+ #
285
+ # @return [String, nil]
286
+ optional :role, String
287
+
288
+ # @!method initialize(content: nil, role: nil)
289
+ # @param content [String] The content of the message.
290
+ #
291
+ # @param role [String] The role of the speaker.
292
+ end
254
293
  end
255
294
 
256
295
  # Run mode for the evaluation. The run mode allows the user to optimize for speed,
@@ -5,7 +5,8 @@ module Deeprails
5
5
  # @see Deeprails::Resources::Monitor#retrieve_event
6
6
  class MonitorEventDetailResponse < Deeprails::Internal::Type::BaseModel
7
7
  # @!attribute capabilities
8
- # The capabilities associated with the monitor event.
8
+ # The extended AI capabilities associated with the monitor event. Can be
9
+ # `web_search`, `file_search`, and/or `context_awareness`.
9
10
  #
10
11
  # @return [Array<Deeprails::Models::MonitorEventDetailResponse::Capability>, nil]
11
12
  optional :capabilities,
@@ -84,7 +85,10 @@ module Deeprails
84
85
  optional :timestamp, Time
85
86
 
86
87
  # @!method initialize(capabilities: nil, eval_time: nil, evaluation_result: nil, event_id: nil, files: nil, guardrail_metrics: nil, model_input: nil, model_output: nil, monitor_id: nil, nametag: nil, run_mode: nil, status: nil, timestamp: nil)
87
- # @param capabilities [Array<Deeprails::Models::MonitorEventDetailResponse::Capability>] The capabilities associated with the monitor event.
88
+ # Some parameter documentations has been truncated, see
89
+ # {Deeprails::Models::MonitorEventDetailResponse} for more details.
90
+ #
91
+ # @param capabilities [Array<Deeprails::Models::MonitorEventDetailResponse::Capability>] The extended AI capabilities associated with the monitor event. Can be `web_sear
88
92
  #
89
93
  # @param eval_time [String] The time spent on the evaluation in seconds.
90
94
  #
@@ -9,8 +9,8 @@ module Deeprails
9
9
 
10
10
  # @!attribute model_input
11
11
  # A dictionary of inputs sent to the LLM to generate output. The dictionary must
12
- # contain at least a `user_prompt` field or a `system_prompt` field. For
13
- # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
12
+ # contain a `user_prompt` field. For ground_truth_adherence guardrail metric,
13
+ # `ground_truth` should be provided.
14
14
  #
15
15
  # @return [Deeprails::Models::MonitorSubmitEventParams::ModelInput]
16
16
  required :model_input, -> { Deeprails::MonitorSubmitEventParams::ModelInput }
@@ -30,8 +30,8 @@ module Deeprails
30
30
  # @!attribute run_mode
31
31
  # Run mode for the monitor event. The run mode allows the user to optimize for
32
32
  # speed, accuracy, and cost by determining which models are used to evaluate the
33
- # event. Available run modes include `precision_plus`, `precision`, `smart`, and
34
- # `economy`. Defaults to `smart`.
33
+ # event. Available run modes include `precision_plus_codex`, `precision_plus`,
34
+ # `precision`, `smart`, and `economy`. Defaults to `smart`.
35
35
  #
36
36
  # @return [Symbol, Deeprails::Models::MonitorSubmitEventParams::RunMode, nil]
37
37
  optional :run_mode, enum: -> { Deeprails::MonitorSubmitEventParams::RunMode }
@@ -51,6 +51,23 @@ module Deeprails
51
51
  # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
52
52
 
53
53
  class ModelInput < Deeprails::Internal::Type::BaseModel
54
+ # @!attribute user_prompt
55
+ # The user prompt used to generate the output.
56
+ #
57
+ # @return [String]
58
+ required :user_prompt, String
59
+
60
+ # @!attribute context
61
+ # Any structured information that directly relates to the model’s input and
62
+ # expected output—e.g., the recent turn-by-turn history between an AI tutor and a
63
+ # student, facts or state passed through an agentic workflow, or other
64
+ # domain-specific signals your system already knows and wants the model to
65
+ # condition on.
66
+ #
67
+ # @return [Array<Deeprails::Models::MonitorSubmitEventParams::ModelInput::Context>, nil]
68
+ optional :context,
69
+ -> { Deeprails::Internal::Type::ArrayOf[Deeprails::MonitorSubmitEventParams::ModelInput::Context] }
70
+
54
71
  # @!attribute ground_truth
55
72
  # The ground truth for evaluating Ground Truth Adherence guardrail.
56
73
  #
@@ -63,31 +80,50 @@ module Deeprails
63
80
  # @return [String, nil]
64
81
  optional :system_prompt, String
65
82
 
66
- # @!attribute user_prompt
67
- # The user prompt used to generate the output.
83
+ # @!method initialize(user_prompt:, context: nil, ground_truth: nil, system_prompt: nil)
84
+ # Some parameter documentations has been truncated, see
85
+ # {Deeprails::Models::MonitorSubmitEventParams::ModelInput} for more details.
68
86
  #
69
- # @return [String, nil]
70
- optional :user_prompt, String
71
-
72
- # @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
73
87
  # A dictionary of inputs sent to the LLM to generate output. The dictionary must
74
- # contain at least a `user_prompt` field or a `system_prompt` field. For
75
- # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
88
+ # contain a `user_prompt` field. For ground_truth_adherence guardrail metric,
89
+ # `ground_truth` should be provided.
90
+ #
91
+ # @param user_prompt [String] The user prompt used to generate the output.
92
+ #
93
+ # @param context [Array<Deeprails::Models::MonitorSubmitEventParams::ModelInput::Context>] Any structured information that directly relates to the model’s input and expect
76
94
  #
77
95
  # @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
78
96
  #
79
97
  # @param system_prompt [String] The system prompt used to generate the output.
80
- #
81
- # @param user_prompt [String] The user prompt used to generate the output.
98
+
99
+ class Context < Deeprails::Internal::Type::BaseModel
100
+ # @!attribute content
101
+ # The content of the message.
102
+ #
103
+ # @return [String, nil]
104
+ optional :content, String
105
+
106
+ # @!attribute role
107
+ # The role of the speaker.
108
+ #
109
+ # @return [String, nil]
110
+ optional :role, String
111
+
112
+ # @!method initialize(content: nil, role: nil)
113
+ # @param content [String] The content of the message.
114
+ #
115
+ # @param role [String] The role of the speaker.
116
+ end
82
117
  end
83
118
 
84
119
  # Run mode for the monitor event. The run mode allows the user to optimize for
85
120
  # speed, accuracy, and cost by determining which models are used to evaluate the
86
- # event. Available run modes include `precision_plus`, `precision`, `smart`, and
87
- # `economy`. Defaults to `smart`.
121
+ # event. Available run modes include `precision_plus_codex`, `precision_plus`,
122
+ # `precision`, `smart`, and `economy`. Defaults to `smart`.
88
123
  module RunMode
89
124
  extend Deeprails::Internal::Type::Enum
90
125
 
126
+ PRECISION_PLUS_CODEX = :precision_plus_codex
91
127
  PRECISION_PLUS = :precision_plus
92
128
  PRECISION = :precision
93
129
  SMART = :smart
@@ -8,13 +8,28 @@ module Deeprails
8
8
  include Deeprails::Internal::Type::RequestParameters
9
9
 
10
10
  # @!attribute description
11
- # Description of the monitor.
11
+ # New description of the monitor.
12
12
  #
13
13
  # @return [String, nil]
14
14
  optional :description, String
15
15
 
16
+ # @!attribute file_search
17
+ # An array of file IDs to search in the monitor's evaluations. Files must be
18
+ # uploaded via the DeepRails API first.
19
+ #
20
+ # @return [Array<String>, nil]
21
+ optional :file_search, Deeprails::Internal::Type::ArrayOf[String]
22
+
23
+ # @!attribute guardrail_metrics
24
+ # An array of the new guardrail metrics that model input and output pairs will be
25
+ # evaluated on.
26
+ #
27
+ # @return [Array<Symbol, Deeprails::Models::MonitorUpdateParams::GuardrailMetric>, nil]
28
+ optional :guardrail_metrics,
29
+ -> { Deeprails::Internal::Type::ArrayOf[enum: Deeprails::MonitorUpdateParams::GuardrailMetric] }
30
+
16
31
  # @!attribute name
17
- # Name of the monitor.
32
+ # New name of the monitor.
18
33
  #
19
34
  # @return [String, nil]
20
35
  optional :name, String
@@ -26,18 +41,44 @@ module Deeprails
26
41
  # @return [Symbol, Deeprails::Models::MonitorUpdateParams::Status, nil]
27
42
  optional :status, enum: -> { Deeprails::MonitorUpdateParams::Status }
28
43
 
29
- # @!method initialize(description: nil, name: nil, status: nil, request_options: {})
44
+ # @!attribute web_search
45
+ # Whether to enable web search for this monitor's evaluations.
46
+ #
47
+ # @return [Boolean, nil]
48
+ optional :web_search, Deeprails::Internal::Type::Boolean
49
+
50
+ # @!method initialize(description: nil, file_search: nil, guardrail_metrics: nil, name: nil, status: nil, web_search: nil, request_options: {})
30
51
  # Some parameter documentations has been truncated, see
31
52
  # {Deeprails::Models::MonitorUpdateParams} for more details.
32
53
  #
33
- # @param description [String] Description of the monitor.
54
+ # @param description [String] New description of the monitor.
55
+ #
56
+ # @param file_search [Array<String>] An array of file IDs to search in the monitor's evaluations. Files must be uploa
57
+ #
58
+ # @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorUpdateParams::GuardrailMetric>] An array of the new guardrail metrics that model input and output pairs will be
34
59
  #
35
- # @param name [String] Name of the monitor.
60
+ # @param name [String] New name of the monitor.
36
61
  #
37
62
  # @param status [Symbol, Deeprails::Models::MonitorUpdateParams::Status] Status of the monitor. Can be `active` or `inactive`. Inactive monitors no lon
38
63
  #
64
+ # @param web_search [Boolean] Whether to enable web search for this monitor's evaluations.
65
+ #
39
66
  # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
40
67
 
68
+ module GuardrailMetric
69
+ extend Deeprails::Internal::Type::Enum
70
+
71
+ CORRECTNESS = :correctness
72
+ COMPLETENESS = :completeness
73
+ INSTRUCTION_ADHERENCE = :instruction_adherence
74
+ CONTEXT_ADHERENCE = :context_adherence
75
+ GROUND_TRUTH_ADHERENCE = :ground_truth_adherence
76
+ COMPREHENSIVE_SAFETY = :comprehensive_safety
77
+
78
+ # @!method self.values
79
+ # @return [Array<Symbol>]
80
+ end
81
+
41
82
  # Status of the monitor. Can be `active` or `inactive`. Inactive monitors no
42
83
  # longer record and evaluate events.
43
84
  module Status