deeprails 0.18.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +92 -0
  3. data/README.md +1 -1
  4. data/lib/deeprails/internal/stream.rb +29 -0
  5. data/lib/deeprails/internal/transport/base_client.rb +3 -1
  6. data/lib/deeprails/internal/transport/pooled_net_requester.rb +13 -11
  7. data/lib/deeprails/internal/type/base_stream.rb +83 -0
  8. data/lib/deeprails/internal/util.rb +39 -3
  9. data/lib/deeprails/models/defend_create_workflow_params.rb +14 -1
  10. data/lib/deeprails/models/defend_response.rb +134 -10
  11. data/lib/deeprails/models/defend_retrieve_event_params.rb +7 -1
  12. data/lib/deeprails/models/defend_retrieve_workflow_params.rb +8 -1
  13. data/lib/deeprails/models/defend_submit_and_stream_event_params.rb +92 -0
  14. data/lib/deeprails/models/defend_submit_and_stream_event_response.rb +7 -0
  15. data/lib/deeprails/models/defend_submit_event_params.rb +70 -20
  16. data/lib/deeprails/models/defend_update_response.rb +9 -1
  17. data/lib/deeprails/models/defend_update_workflow_params.rb +138 -5
  18. data/lib/deeprails/models/monitor_create_params.rb +14 -1
  19. data/lib/deeprails/models/monitor_detail_response.rb +60 -17
  20. data/lib/deeprails/models/monitor_event_detail_response.rb +11 -5
  21. data/lib/deeprails/models/monitor_retrieve_event_params.rb +7 -1
  22. data/lib/deeprails/models/monitor_retrieve_params.rb +8 -1
  23. data/lib/deeprails/models/monitor_submit_event_params.rb +70 -20
  24. data/lib/deeprails/models/monitor_update_params.rb +53 -5
  25. data/lib/deeprails/models/workflow_event_detail_response.rb +99 -16
  26. data/lib/deeprails/models/workflow_event_response.rb +9 -1
  27. data/lib/deeprails/models.rb +2 -0
  28. data/lib/deeprails/resources/defend.rb +70 -5
  29. data/lib/deeprails/resources/monitor.rb +16 -7
  30. data/lib/deeprails/version.rb +1 -1
  31. data/lib/deeprails.rb +5 -0
  32. data/manifest.yaml +1 -0
  33. data/rbi/deeprails/internal/stream.rbi +20 -0
  34. data/rbi/deeprails/internal/transport/base_client.rbi +18 -2
  35. data/rbi/deeprails/internal/type/base_stream.rbi +75 -0
  36. data/rbi/deeprails/internal/util.rbi +21 -1
  37. data/rbi/deeprails/models/defend_create_workflow_params.rbi +21 -0
  38. data/rbi/deeprails/models/defend_response.rbi +286 -14
  39. data/rbi/deeprails/models/defend_retrieve_event_params.rbi +10 -2
  40. data/rbi/deeprails/models/defend_retrieve_workflow_params.rbi +10 -1
  41. data/rbi/deeprails/models/defend_submit_and_stream_event_params.rbi +155 -0
  42. data/rbi/deeprails/models/defend_submit_and_stream_event_response.rbi +7 -0
  43. data/rbi/deeprails/models/defend_submit_event_params.rbi +134 -31
  44. data/rbi/deeprails/models/defend_update_response.rbi +14 -3
  45. data/rbi/deeprails/models/defend_update_workflow_params.rbi +304 -4
  46. data/rbi/deeprails/models/monitor_create_params.rbi +21 -0
  47. data/rbi/deeprails/models/monitor_detail_response.rbi +122 -28
  48. data/rbi/deeprails/models/monitor_event_detail_response.rbi +20 -8
  49. data/rbi/deeprails/models/monitor_retrieve_event_params.rbi +10 -2
  50. data/rbi/deeprails/models/monitor_retrieve_params.rbi +10 -1
  51. data/rbi/deeprails/models/monitor_submit_event_params.rbi +134 -31
  52. data/rbi/deeprails/models/monitor_update_params.rbi +113 -4
  53. data/rbi/deeprails/models/workflow_event_detail_response.rbi +204 -24
  54. data/rbi/deeprails/models/workflow_event_response.rbi +8 -0
  55. data/rbi/deeprails/models.rbi +3 -0
  56. data/rbi/deeprails/resources/defend.rbi +101 -6
  57. data/rbi/deeprails/resources/monitor.rbi +31 -8
  58. data/sig/deeprails/internal/stream.rbs +9 -0
  59. data/sig/deeprails/internal/type/base_stream.rbs +38 -0
  60. data/sig/deeprails/internal/util.rbs +10 -0
  61. data/sig/deeprails/models/defend_create_workflow_params.rbs +7 -0
  62. data/sig/deeprails/models/defend_response.rbs +114 -8
  63. data/sig/deeprails/models/defend_retrieve_event_params.rbs +6 -1
  64. data/sig/deeprails/models/defend_retrieve_workflow_params.rbs +6 -1
  65. data/sig/deeprails/models/defend_submit_and_stream_event_params.rbs +73 -0
  66. data/sig/deeprails/models/defend_submit_and_stream_event_response.rbs +5 -0
  67. data/sig/deeprails/models/defend_submit_event_params.rbs +53 -13
  68. data/sig/deeprails/models/defend_update_response.rbs +10 -3
  69. data/sig/deeprails/models/defend_update_workflow_params.rbs +108 -1
  70. data/sig/deeprails/models/monitor_create_params.rbs +7 -0
  71. data/sig/deeprails/models/monitor_detail_response.rbs +48 -13
  72. data/sig/deeprails/models/monitor_event_detail_response.rbs +12 -4
  73. data/sig/deeprails/models/monitor_retrieve_event_params.rbs +6 -1
  74. data/sig/deeprails/models/monitor_retrieve_params.rbs +6 -1
  75. data/sig/deeprails/models/monitor_submit_event_params.rbs +53 -13
  76. data/sig/deeprails/models/monitor_update_params.rbs +50 -1
  77. data/sig/deeprails/models/workflow_event_detail_response.rbs +100 -15
  78. data/sig/deeprails/models/workflow_event_response.rbs +5 -0
  79. data/sig/deeprails/models.rbs +2 -0
  80. data/sig/deeprails/resources/defend.rbs +20 -0
  81. data/sig/deeprails/resources/monitor.rbs +4 -0
  82. metadata +28 -2
@@ -7,6 +7,11 @@ module Deeprails
7
7
  extend Deeprails::Internal::Type::RequestParameters::Converter
8
8
  include Deeprails::Internal::Type::RequestParameters
9
9
 
10
+ # @!attribute monitor_id
11
+ #
12
+ # @return [String]
13
+ required :monitor_id, String
14
+
10
15
  # @!attribute limit
11
16
  # Limit the number of returned evaluations associated with this monitor. Defaults
12
17
  # to 10.
@@ -14,10 +19,12 @@ module Deeprails
14
19
  # @return [Integer, nil]
15
20
  optional :limit, Integer
16
21
 
17
- # @!method initialize(limit: nil, request_options: {})
22
+ # @!method initialize(monitor_id:, limit: nil, request_options: {})
18
23
  # Some parameter documentations has been truncated, see
19
24
  # {Deeprails::Models::MonitorRetrieveParams} for more details.
20
25
  #
26
+ # @param monitor_id [String]
27
+ #
21
28
  # @param limit [Integer] Limit the number of returned evaluations associated with this monitor. Defaults
22
29
  #
23
30
  # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
@@ -7,10 +7,15 @@ module Deeprails
7
7
  extend Deeprails::Internal::Type::RequestParameters::Converter
8
8
  include Deeprails::Internal::Type::RequestParameters
9
9
 
10
+ # @!attribute monitor_id
11
+ #
12
+ # @return [String]
13
+ required :monitor_id, String
14
+
10
15
  # @!attribute model_input
11
16
  # A dictionary of inputs sent to the LLM to generate output. The dictionary must
12
- # contain at least a `user_prompt` field or a `system_prompt` field. For
13
- # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
17
+ # contain a `user_prompt` field. For ground_truth_adherence guardrail metric,
18
+ # `ground_truth` should be provided.
14
19
  #
15
20
  # @return [Deeprails::Models::MonitorSubmitEventParams::ModelInput]
16
21
  required :model_input, -> { Deeprails::MonitorSubmitEventParams::ModelInput }
@@ -30,16 +35,21 @@ module Deeprails
30
35
  # @!attribute run_mode
31
36
  # Run mode for the monitor event. The run mode allows the user to optimize for
32
37
  # speed, accuracy, and cost by determining which models are used to evaluate the
33
- # event. Available run modes include `precision_plus`, `precision`, `smart`, and
34
- # `economy`. Defaults to `smart`.
38
+ # event. Available run modes (fastest to most thorough): `super_fast`, `fast`,
39
+ # `precision`, `precision_codex`, `precision_max`, and `precision_max_codex`.
40
+ # Defaults to `fast`. Note: `super_fast` does not support Web Search or File
41
+ # Search — if your monitor has these capabilities enabled, use a different run
42
+ # mode or edit the monitor to disable them.
35
43
  #
36
44
  # @return [Symbol, Deeprails::Models::MonitorSubmitEventParams::RunMode, nil]
37
45
  optional :run_mode, enum: -> { Deeprails::MonitorSubmitEventParams::RunMode }
38
46
 
39
- # @!method initialize(model_input:, model_output:, nametag: nil, run_mode: nil, request_options: {})
47
+ # @!method initialize(monitor_id:, model_input:, model_output:, nametag: nil, run_mode: nil, request_options: {})
40
48
  # Some parameter documentations has been truncated, see
41
49
  # {Deeprails::Models::MonitorSubmitEventParams} for more details.
42
50
  #
51
+ # @param monitor_id [String]
52
+ #
43
53
  # @param model_input [Deeprails::Models::MonitorSubmitEventParams::ModelInput] A dictionary of inputs sent to the LLM to generate output. The dictionary must c
44
54
  #
45
55
  # @param model_output [String] Output generated by the LLM to be evaluated.
@@ -51,6 +61,23 @@ module Deeprails
51
61
  # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
52
62
 
53
63
  class ModelInput < Deeprails::Internal::Type::BaseModel
64
+ # @!attribute user_prompt
65
+ # The user prompt used to generate the output.
66
+ #
67
+ # @return [String]
68
+ required :user_prompt, String
69
+
70
+ # @!attribute context
71
+ # Any structured information that directly relates to the model’s input and
72
+ # expected output—e.g., the recent turn-by-turn history between an AI tutor and a
73
+ # student, facts or state passed through an agentic workflow, or other
74
+ # domain-specific signals your system already knows and wants the model to
75
+ # condition on.
76
+ #
77
+ # @return [Array<Deeprails::Models::MonitorSubmitEventParams::ModelInput::Context>, nil]
78
+ optional :context,
79
+ -> { Deeprails::Internal::Type::ArrayOf[Deeprails::MonitorSubmitEventParams::ModelInput::Context] }
80
+
54
81
  # @!attribute ground_truth
55
82
  # The ground truth for evaluating Ground Truth Adherence guardrail.
56
83
  #
@@ -63,35 +90,58 @@ module Deeprails
63
90
  # @return [String, nil]
64
91
  optional :system_prompt, String
65
92
 
66
- # @!attribute user_prompt
67
- # The user prompt used to generate the output.
93
+ # @!method initialize(user_prompt:, context: nil, ground_truth: nil, system_prompt: nil)
94
+ # Some parameter documentations has been truncated, see
95
+ # {Deeprails::Models::MonitorSubmitEventParams::ModelInput} for more details.
68
96
  #
69
- # @return [String, nil]
70
- optional :user_prompt, String
71
-
72
- # @!method initialize(ground_truth: nil, system_prompt: nil, user_prompt: nil)
73
97
  # A dictionary of inputs sent to the LLM to generate output. The dictionary must
74
- # contain at least a `user_prompt` field or a `system_prompt` field. For
75
- # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
98
+ # contain a `user_prompt` field. For ground_truth_adherence guardrail metric,
99
+ # `ground_truth` should be provided.
100
+ #
101
+ # @param user_prompt [String] The user prompt used to generate the output.
102
+ #
103
+ # @param context [Array<Deeprails::Models::MonitorSubmitEventParams::ModelInput::Context>] Any structured information that directly relates to the model’s input and expect
76
104
  #
77
105
  # @param ground_truth [String] The ground truth for evaluating Ground Truth Adherence guardrail.
78
106
  #
79
107
  # @param system_prompt [String] The system prompt used to generate the output.
80
- #
81
- # @param user_prompt [String] The user prompt used to generate the output.
108
+
109
+ class Context < Deeprails::Internal::Type::BaseModel
110
+ # @!attribute content
111
+ # The content of the message.
112
+ #
113
+ # @return [String, nil]
114
+ optional :content, String
115
+
116
+ # @!attribute role
117
+ # The role of the speaker.
118
+ #
119
+ # @return [String, nil]
120
+ optional :role, String
121
+
122
+ # @!method initialize(content: nil, role: nil)
123
+ # @param content [String] The content of the message.
124
+ #
125
+ # @param role [String] The role of the speaker.
126
+ end
82
127
  end
83
128
 
84
129
  # Run mode for the monitor event. The run mode allows the user to optimize for
85
130
  # speed, accuracy, and cost by determining which models are used to evaluate the
86
- # event. Available run modes include `precision_plus`, `precision`, `smart`, and
87
- # `economy`. Defaults to `smart`.
131
+ # event. Available run modes (fastest to most thorough): `super_fast`, `fast`,
132
+ # `precision`, `precision_codex`, `precision_max`, and `precision_max_codex`.
133
+ # Defaults to `fast`. Note: `super_fast` does not support Web Search or File
134
+ # Search — if your monitor has these capabilities enabled, use a different run
135
+ # mode or edit the monitor to disable them.
88
136
  module RunMode
89
137
  extend Deeprails::Internal::Type::Enum
90
138
 
91
- PRECISION_PLUS = :precision_plus
139
+ SUPER_FAST = :super_fast
140
+ FAST = :fast
92
141
  PRECISION = :precision
93
- SMART = :smart
94
- ECONOMY = :economy
142
+ PRECISION_CODEX = :precision_codex
143
+ PRECISION_MAX = :precision_max
144
+ PRECISION_MAX_CODEX = :precision_max_codex
95
145
 
96
146
  # @!method self.values
97
147
  # @return [Array<Symbol>]
@@ -7,14 +7,34 @@ module Deeprails
7
7
  extend Deeprails::Internal::Type::RequestParameters::Converter
8
8
  include Deeprails::Internal::Type::RequestParameters
9
9
 
10
+ # @!attribute monitor_id
11
+ #
12
+ # @return [String]
13
+ required :monitor_id, String
14
+
10
15
  # @!attribute description
11
- # Description of the monitor.
16
+ # New description of the monitor.
12
17
  #
13
18
  # @return [String, nil]
14
19
  optional :description, String
15
20
 
21
+ # @!attribute file_search
22
+ # An array of file IDs to search in the monitor's evaluations. Files must be
23
+ # uploaded via the DeepRails API first.
24
+ #
25
+ # @return [Array<String>, nil]
26
+ optional :file_search, Deeprails::Internal::Type::ArrayOf[String]
27
+
28
+ # @!attribute guardrail_metrics
29
+ # An array of the new guardrail metrics that model input and output pairs will be
30
+ # evaluated on.
31
+ #
32
+ # @return [Array<Symbol, Deeprails::Models::MonitorUpdateParams::GuardrailMetric>, nil]
33
+ optional :guardrail_metrics,
34
+ -> { Deeprails::Internal::Type::ArrayOf[enum: Deeprails::MonitorUpdateParams::GuardrailMetric] }
35
+
16
36
  # @!attribute name
17
- # Name of the monitor.
37
+ # New name of the monitor.
18
38
  #
19
39
  # @return [String, nil]
20
40
  optional :name, String
@@ -26,18 +46,46 @@ module Deeprails
26
46
  # @return [Symbol, Deeprails::Models::MonitorUpdateParams::Status, nil]
27
47
  optional :status, enum: -> { Deeprails::MonitorUpdateParams::Status }
28
48
 
29
- # @!method initialize(description: nil, name: nil, status: nil, request_options: {})
49
+ # @!attribute web_search
50
+ # Whether to enable web search for this monitor's evaluations.
51
+ #
52
+ # @return [Boolean, nil]
53
+ optional :web_search, Deeprails::Internal::Type::Boolean
54
+
55
+ # @!method initialize(monitor_id:, description: nil, file_search: nil, guardrail_metrics: nil, name: nil, status: nil, web_search: nil, request_options: {})
30
56
  # Some parameter documentations has been truncated, see
31
57
  # {Deeprails::Models::MonitorUpdateParams} for more details.
32
58
  #
33
- # @param description [String] Description of the monitor.
59
+ # @param monitor_id [String]
60
+ #
61
+ # @param description [String] New description of the monitor.
62
+ #
63
+ # @param file_search [Array<String>] An array of file IDs to search in the monitor's evaluations. Files must be uploa
34
64
  #
35
- # @param name [String] Name of the monitor.
65
+ # @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorUpdateParams::GuardrailMetric>] An array of the new guardrail metrics that model input and output pairs will be
66
+ #
67
+ # @param name [String] New name of the monitor.
36
68
  #
37
69
  # @param status [Symbol, Deeprails::Models::MonitorUpdateParams::Status] Status of the monitor. Can be `active` or `inactive`. Inactive monitors no lon
38
70
  #
71
+ # @param web_search [Boolean] Whether to enable web search for this monitor's evaluations.
72
+ #
39
73
  # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
40
74
 
75
+ module GuardrailMetric
76
+ extend Deeprails::Internal::Type::Enum
77
+
78
+ CORRECTNESS = :correctness
79
+ COMPLETENESS = :completeness
80
+ INSTRUCTION_ADHERENCE = :instruction_adherence
81
+ CONTEXT_ADHERENCE = :context_adherence
82
+ GROUND_TRUTH_ADHERENCE = :ground_truth_adherence
83
+ COMPREHENSIVE_SAFETY = :comprehensive_safety
84
+
85
+ # @!method self.values
86
+ # @return [Array<Symbol>]
87
+ end
88
+
41
89
  # Status of the monitor. Can be `active` or `inactive`. Inactive monitors no
42
90
  # longer record and evaluate events.
43
91
  module Status
@@ -4,6 +4,11 @@ module Deeprails
4
4
  module Models
5
5
  # @see Deeprails::Resources::Defend#retrieve_event
6
6
  class WorkflowEventDetailResponse < Deeprails::Internal::Type::BaseModel
7
+ # @!attribute analysis_of_failures
8
+ #
9
+ # @return [Array<String>]
10
+ required :analysis_of_failures, Deeprails::Internal::Type::ArrayOf[String]
11
+
7
12
  # @!attribute evaluation_history
8
13
  # History of evaluations for the event.
9
14
  #
@@ -44,13 +49,24 @@ module Deeprails
44
49
  required :improvement_action, enum: -> { Deeprails::WorkflowEventDetailResponse::ImprovementAction }
45
50
 
46
51
  # @!attribute improvement_tool_status
47
- # Status of the improvement tool used to improve the event.
52
+ # Status of the improvement tool used to improve the event. `improvement_required`
53
+ # indicates that the evaluation is complete and the improvement action is needed
54
+ # but is not taking place. `improved` and `improvement_failed` indicate when the
55
+ # improvement action concludes, successfully and unsuccessfully, respectively.
56
+ # `no_improvement_required` means that the first evaluation passed all its
57
+ # metrics!
48
58
  #
49
59
  # @return [Symbol, Deeprails::Models::WorkflowEventDetailResponse::ImprovementToolStatus, nil]
50
60
  required :improvement_tool_status,
51
61
  enum: -> { Deeprails::WorkflowEventDetailResponse::ImprovementToolStatus },
52
62
  nil?: true
53
63
 
64
+ # @!attribute key_improvements
65
+ #
66
+ # @return [Array<Deeprails::Models::WorkflowEventDetailResponse::KeyImprovement>]
67
+ required :key_improvements,
68
+ -> { Deeprails::Internal::Type::ArrayOf[Deeprails::WorkflowEventDetailResponse::KeyImprovement] }
69
+
54
70
  # @!attribute status
55
71
  # Status of the event.
56
72
  #
@@ -78,8 +94,8 @@ module Deeprails
78
94
  -> { Deeprails::Internal::Type::HashOf[enum: Deeprails::WorkflowEventDetailResponse::AutomaticHallucinationToleranceLevel] }
79
95
 
80
96
  # @!attribute capabilities
81
- # Extended AI capabilities available to the event, if any. Can be `web_search`
82
- # and/or `file_search`.
97
+ # Extended AI capabilities available to the event, if any. Can be `web_search`,
98
+ # `file_search`, and/or `context_awareness`.
83
99
  #
84
100
  # @return [Array<Deeprails::Models::WorkflowEventDetailResponse::Capability>, nil]
85
101
  optional :capabilities,
@@ -99,10 +115,19 @@ module Deeprails
99
115
  # @return [Array<Deeprails::Models::WorkflowEventDetailResponse::File>, nil]
100
116
  optional :files, -> { Deeprails::Internal::Type::ArrayOf[Deeprails::WorkflowEventDetailResponse::File] }
101
117
 
102
- # @!method initialize(evaluation_history:, evaluation_result:, event_id:, filtered:, improved_model_output:, improvement_action:, improvement_tool_status:, status:, threshold_type:, workflow_id:, automatic_hallucination_tolerance_levels: nil, capabilities: nil, custom_hallucination_threshold_values: nil, files: nil)
118
+ # @!attribute max_improvement_attempts
119
+ # The maximum number of improvement attempts to be applied to one event before it
120
+ # is considered failed.
121
+ #
122
+ # @return [Integer, nil]
123
+ optional :max_improvement_attempts, Integer
124
+
125
+ # @!method initialize(analysis_of_failures:, evaluation_history:, evaluation_result:, event_id:, filtered:, improved_model_output:, improvement_action:, improvement_tool_status:, key_improvements:, status:, threshold_type:, workflow_id:, automatic_hallucination_tolerance_levels: nil, capabilities: nil, custom_hallucination_threshold_values: nil, files: nil, max_improvement_attempts: nil)
103
126
  # Some parameter documentations has been truncated, see
104
127
  # {Deeprails::Models::WorkflowEventDetailResponse} for more details.
105
128
  #
129
+ # @param analysis_of_failures [Array<String>]
130
+ #
106
131
  # @param evaluation_history [Array<Deeprails::Models::WorkflowEventDetailResponse::EvaluationHistory>] History of evaluations for the event.
107
132
  #
108
133
  # @param evaluation_result [Hash{Symbol=>Object}] Evaluation result consisting of average scores and rationales for each of the ev
@@ -115,7 +140,9 @@ module Deeprails
115
140
  #
116
141
  # @param improvement_action [Symbol, Deeprails::Models::WorkflowEventDetailResponse::ImprovementAction] Type of improvement action used to improve the event.
117
142
  #
118
- # @param improvement_tool_status [Symbol, Deeprails::Models::WorkflowEventDetailResponse::ImprovementToolStatus, nil] Status of the improvement tool used to improve the event.
143
+ # @param improvement_tool_status [Symbol, Deeprails::Models::WorkflowEventDetailResponse::ImprovementToolStatus, nil] Status of the improvement tool used to improve the event. `improvement_required`
144
+ #
145
+ # @param key_improvements [Array<Deeprails::Models::WorkflowEventDetailResponse::KeyImprovement>]
119
146
  #
120
147
  # @param status [Symbol, Deeprails::Models::WorkflowEventDetailResponse::Status] Status of the event.
121
148
  #
@@ -125,13 +152,20 @@ module Deeprails
125
152
  #
126
153
  # @param automatic_hallucination_tolerance_levels [Hash{Symbol=>Symbol, Deeprails::Models::WorkflowEventDetailResponse::AutomaticHallucinationToleranceLevel}] Mapping of guardrail metric names to tolerance values. Values are strings (`low`
127
154
  #
128
- # @param capabilities [Array<Deeprails::Models::WorkflowEventDetailResponse::Capability>] Extended AI capabilities available to the event, if any. Can be `web_search` and
155
+ # @param capabilities [Array<Deeprails::Models::WorkflowEventDetailResponse::Capability>] Extended AI capabilities available to the event, if any. Can be `web_search`, `f
129
156
  #
130
157
  # @param custom_hallucination_threshold_values [Hash{Symbol=>Float}] Mapping of guardrail metric names to threshold values. Values are floating point
131
158
  #
132
159
  # @param files [Array<Deeprails::Models::WorkflowEventDetailResponse::File>] List of files available to the event, if any. Will only be present if `file_sear
160
+ #
161
+ # @param max_improvement_attempts [Integer] The maximum number of improvement attempts to be applied to one event before it
133
162
 
134
163
  class EvaluationHistory < Deeprails::Internal::Type::BaseModel
164
+ # @!attribute analysis_of_failures
165
+ #
166
+ # @return [String, nil]
167
+ optional :analysis_of_failures, String
168
+
135
169
  # @!attribute attempt
136
170
  #
137
171
  # @return [String, nil]
@@ -167,6 +201,17 @@ module Deeprails
167
201
  # @return [Array<String>, nil]
168
202
  optional :guardrail_metrics, Deeprails::Internal::Type::ArrayOf[String]
169
203
 
204
+ # @!attribute improvement_tool_status
205
+ #
206
+ # @return [Symbol, Deeprails::Models::WorkflowEventDetailResponse::EvaluationHistory::ImprovementToolStatus, nil]
207
+ optional :improvement_tool_status,
208
+ enum: -> { Deeprails::WorkflowEventDetailResponse::EvaluationHistory::ImprovementToolStatus }
209
+
210
+ # @!attribute key_improvements
211
+ #
212
+ # @return [Array<String>, nil]
213
+ optional :key_improvements, Deeprails::Internal::Type::ArrayOf[String]
214
+
170
215
  # @!attribute model_input
171
216
  #
172
217
  # @return [Hash{Symbol=>Object}, nil]
@@ -177,11 +222,6 @@ module Deeprails
177
222
  # @return [String, nil]
178
223
  optional :model_output, String
179
224
 
180
- # @!attribute modified_at
181
- #
182
- # @return [Time, nil]
183
- optional :modified_at, Time
184
-
185
225
  # @!attribute nametag
186
226
  #
187
227
  # @return [String, nil]
@@ -197,7 +237,8 @@ module Deeprails
197
237
  # @return [String, nil]
198
238
  optional :run_mode, String
199
239
 
200
- # @!method initialize(attempt: nil, created_at: nil, error_message: nil, evaluation_result: nil, evaluation_status: nil, evaluation_total_cost: nil, guardrail_metrics: nil, model_input: nil, model_output: nil, modified_at: nil, nametag: nil, progress: nil, run_mode: nil)
240
+ # @!method initialize(analysis_of_failures: nil, attempt: nil, created_at: nil, error_message: nil, evaluation_result: nil, evaluation_status: nil, evaluation_total_cost: nil, guardrail_metrics: nil, improvement_tool_status: nil, key_improvements: nil, model_input: nil, model_output: nil, nametag: nil, progress: nil, run_mode: nil)
241
+ # @param analysis_of_failures [String]
201
242
  # @param attempt [String]
202
243
  # @param created_at [Time]
203
244
  # @param error_message [String]
@@ -205,12 +246,26 @@ module Deeprails
205
246
  # @param evaluation_status [String]
206
247
  # @param evaluation_total_cost [Float]
207
248
  # @param guardrail_metrics [Array<String>]
249
+ # @param improvement_tool_status [Symbol, Deeprails::Models::WorkflowEventDetailResponse::EvaluationHistory::ImprovementToolStatus]
250
+ # @param key_improvements [Array<String>]
208
251
  # @param model_input [Hash{Symbol=>Object}]
209
252
  # @param model_output [String]
210
- # @param modified_at [Time]
211
253
  # @param nametag [String]
212
254
  # @param progress [Integer]
213
255
  # @param run_mode [String]
256
+
257
+ # @see Deeprails::Models::WorkflowEventDetailResponse::EvaluationHistory#improvement_tool_status
258
+ module ImprovementToolStatus
259
+ extend Deeprails::Internal::Type::Enum
260
+
261
+ IMPROVED = :improved
262
+ IMPROVEMENT_FAILED = :improvement_failed
263
+ NO_IMPROVEMENT_REQUIRED = :no_improvement_required
264
+ IMPROVEMENT_REQUIRED = :improvement_required
265
+
266
+ # @!method self.values
267
+ # @return [Array<Symbol>]
268
+ end
214
269
  end
215
270
 
216
271
  # Type of improvement action used to improve the event.
@@ -227,20 +282,36 @@ module Deeprails
227
282
  # @return [Array<Symbol>]
228
283
  end
229
284
 
230
- # Status of the improvement tool used to improve the event.
285
+ # Status of the improvement tool used to improve the event. `improvement_required`
286
+ # indicates that the evaluation is complete and the improvement action is needed
287
+ # but is not taking place. `improved` and `improvement_failed` indicate when the
288
+ # improvement action concludes, successfully and unsuccessfully, respectively.
289
+ # `no_improvement_required` means that the first evaluation passed all its
290
+ # metrics!
231
291
  #
232
292
  # @see Deeprails::Models::WorkflowEventDetailResponse#improvement_tool_status
233
293
  module ImprovementToolStatus
234
294
  extend Deeprails::Internal::Type::Enum
235
295
 
236
296
  IMPROVED = :improved
237
- FAILED_ON_MAX_RETRIES = :"failed on max retries"
297
+ IMPROVEMENT_FAILED = :improvement_failed
298
+ NO_IMPROVEMENT_REQUIRED = :no_improvement_required
238
299
  IMPROVEMENT_REQUIRED = :improvement_required
239
300
 
240
301
  # @!method self.values
241
302
  # @return [Array<Symbol>]
242
303
  end
243
304
 
305
+ class KeyImprovement < Deeprails::Internal::Type::BaseModel
306
+ # @!attribute key_improvement
307
+ #
308
+ # @return [Array<String>, nil]
309
+ optional :key_improvement, Deeprails::Internal::Type::ArrayOf[String]
310
+
311
+ # @!method initialize(key_improvement: nil)
312
+ # @param key_improvement [Array<String>]
313
+ end
314
+
244
315
  # Status of the event.
245
316
  #
246
317
  # @see Deeprails::Models::WorkflowEventDetailResponse#status
@@ -304,10 +375,22 @@ module Deeprails
304
375
  # @return [Integer, nil]
305
376
  optional :file_size, Integer
306
377
 
307
- # @!method initialize(file_id: nil, file_name: nil, file_size: nil)
378
+ # @!attribute presigned_url
379
+ #
380
+ # @return [String, nil]
381
+ optional :presigned_url, String
382
+
383
+ # @!attribute presigned_url_expires_at
384
+ #
385
+ # @return [Time, nil]
386
+ optional :presigned_url_expires_at, Time
387
+
388
+ # @!method initialize(file_id: nil, file_name: nil, file_size: nil, presigned_url: nil, presigned_url_expires_at: nil)
308
389
  # @param file_id [String]
309
390
  # @param file_name [String]
310
391
  # @param file_size [Integer]
392
+ # @param presigned_url [String]
393
+ # @param presigned_url_expires_at [Time]
311
394
  end
312
395
  end
313
396
  end
@@ -4,6 +4,12 @@ module Deeprails
4
4
  module Models
5
5
  # @see Deeprails::Resources::Defend#submit_event
6
6
  class WorkflowEventResponse < Deeprails::Internal::Type::BaseModel
7
+ # @!attribute billing_request_id
8
+ # The ID of the billing request for the event.
9
+ #
10
+ # @return [String]
11
+ required :billing_request_id, String
12
+
7
13
  # @!attribute created_at
8
14
  # The time the event was created in UTC.
9
15
  #
@@ -28,7 +34,9 @@ module Deeprails
28
34
  # @return [String]
29
35
  required :workflow_id, String
30
36
 
31
- # @!method initialize(created_at:, event_id:, status:, workflow_id:)
37
+ # @!method initialize(billing_request_id:, created_at:, event_id:, status:, workflow_id:)
38
+ # @param billing_request_id [String] The ID of the billing request for the event.
39
+ #
32
40
  # @param created_at [Time] The time the event was created in UTC.
33
41
  #
34
42
  # @param event_id [String] A unique workflow event ID.
@@ -49,6 +49,8 @@ module Deeprails
49
49
 
50
50
  DefendRetrieveWorkflowParams = Deeprails::Models::DefendRetrieveWorkflowParams
51
51
 
52
+ DefendSubmitAndStreamEventParams = Deeprails::Models::DefendSubmitAndStreamEventParams
53
+
52
54
  DefendSubmitEventParams = Deeprails::Models::DefendSubmitEventParams
53
55
 
54
56
  DefendUpdateResponse = Deeprails::Models::DefendUpdateResponse
@@ -9,7 +9,7 @@ module Deeprails
9
9
  # Use this endpoint to create a new guardrail workflow by specifying guardrail
10
10
  # thresholds, an improvement action, and optional extended capabilities.
11
11
  #
12
- # @overload create_workflow(improvement_action:, name:, threshold_type:, automatic_hallucination_tolerance_levels: nil, custom_hallucination_threshold_values: nil, description: nil, file_search: nil, max_improvement_attempts: nil, web_search: nil, request_options: {})
12
+ # @overload create_workflow(improvement_action:, name:, threshold_type:, automatic_hallucination_tolerance_levels: nil, context_awareness: nil, custom_hallucination_threshold_values: nil, description: nil, file_search: nil, max_improvement_attempts: nil, web_search: nil, request_options: {})
13
13
  #
14
14
  # @param improvement_action [Symbol, Deeprails::Models::DefendCreateWorkflowParams::ImprovementAction] The action used to improve outputs that fail one or more guardrail metrics for t
15
15
  #
@@ -19,6 +19,8 @@ module Deeprails
19
19
  #
20
20
  # @param automatic_hallucination_tolerance_levels [Hash{Symbol=>Symbol, Deeprails::Models::DefendCreateWorkflowParams::AutomaticHallucinationToleranceLevel}] Mapping of guardrail metrics to hallucination tolerance levels
21
21
  #
22
+ # @param context_awareness [Boolean] Context includes any structured information that directly relates to the model’s
23
+ #
22
24
  # @param custom_hallucination_threshold_values [Hash{Symbol=>Float}] Mapping of guardrail metrics to floating point threshold values. Possible metric
23
25
  #
24
26
  # @param description [String] Description for the workflow.
@@ -90,15 +92,59 @@ module Deeprails
90
92
  # @see Deeprails::Models::DefendRetrieveWorkflowParams
91
93
  def retrieve_workflow(workflow_id, params = {})
92
94
  parsed, options = Deeprails::DefendRetrieveWorkflowParams.dump_request(params)
95
+ query = Deeprails::Internal::Util.encode_query_params(parsed)
93
96
  @client.request(
94
97
  method: :get,
95
98
  path: ["defend/%1$s", workflow_id],
96
- query: parsed,
99
+ query: query,
97
100
  model: Deeprails::DefendResponse,
98
101
  options: options
99
102
  )
100
103
  end
101
104
 
105
+ # Some parameter documentations has been truncated, see
106
+ # {Deeprails::Models::DefendSubmitAndStreamEventParams} for more details.
107
+ #
108
+ # Use this endpoint to submit a model input and output pair to a workflow for
109
+ # evaluation with streaming responses.
110
+ #
111
+ # @overload submit_and_stream_event_streaming(workflow_id, model_input:, model_output:, model_used:, run_mode:, stream: nil, nametag: nil, request_options: {})
112
+ #
113
+ # @param workflow_id [String] Path param: The ID of the workflow to create the event for.
114
+ #
115
+ # @param model_input [Hash{Symbol=>Object}] Body param: The input provided to the model (e.g., prompt, messages).
116
+ #
117
+ # @param model_output [String] Body param: The output generated by the model to be evaluated.
118
+ #
119
+ # @param model_used [String] Body param: The model that generated the output (e.g., "gpt-4", "claude-3").
120
+ #
121
+ # @param run_mode [Symbol, Deeprails::Models::DefendSubmitAndStreamEventParams::RunMode] Body param: The evaluation run mode. Streaming is supported on all run modes exc
122
+ #
123
+ # @param stream [Boolean] Query param: Enable SSE streaming for real-time token feedback. Supported on all
124
+ #
125
+ # @param nametag [String] Body param: Optional tag to identify this event.
126
+ #
127
+ # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
128
+ #
129
+ # @return [Deeprails::Internal::Stream<String>]
130
+ #
131
+ # @see Deeprails::Models::DefendSubmitAndStreamEventParams
132
+ def submit_and_stream_event_streaming(workflow_id, params)
133
+ query_params = [:stream]
134
+ parsed, options = Deeprails::DefendSubmitAndStreamEventParams.dump_request(params)
135
+ query = Deeprails::Internal::Util.encode_query_params(parsed.slice(*query_params))
136
+ @client.request(
137
+ method: :post,
138
+ path: ["defend/%1$s/events?stream=true", workflow_id],
139
+ query: query,
140
+ headers: {"accept" => "text/event-stream"},
141
+ body: parsed.except(*query_params),
142
+ stream: Deeprails::Internal::Stream,
143
+ model: String,
144
+ options: options
145
+ )
146
+ end
147
+
102
148
  # Some parameter documentations has been truncated, see
103
149
  # {Deeprails::Models::DefendSubmitEventParams} for more details.
104
150
  #
@@ -135,15 +181,34 @@ module Deeprails
135
181
  )
136
182
  end
137
183
 
184
+ # Some parameter documentations has been truncated, see
185
+ # {Deeprails::Models::DefendUpdateWorkflowParams} for more details.
186
+ #
138
187
  # Use this endpoint to update an existing defend workflow if its details change.
139
188
  #
140
- # @overload update_workflow(workflow_id, description: nil, name: nil, request_options: {})
189
+ # @overload update_workflow(workflow_id, automatic_hallucination_tolerance_levels: nil, context_awareness: nil, custom_hallucination_threshold_values: nil, description: nil, file_search: nil, improvement_action: nil, max_improvement_attempts: nil, name: nil, threshold_type: nil, web_search: nil, request_options: {})
141
190
  #
142
191
  # @param workflow_id [String] The ID of the workflow to edit.
143
192
  #
144
- # @param description [String] Description for the workflow.
193
+ # @param automatic_hallucination_tolerance_levels [Hash{Symbol=>Symbol, Deeprails::Models::DefendUpdateWorkflowParams::AutomaticHallucinationToleranceLevel}] New mapping of guardrail metrics to hallucination tolerance levels
145
194
  #
146
- # @param name [String] Name of the workflow.
195
+ # @param context_awareness [Boolean] Whether to enable context awareness for this workflow's evaluations.
196
+ #
197
+ # @param custom_hallucination_threshold_values [Hash{Symbol=>Float}] New mapping of guardrail metrics to floating point threshold values to be used w
198
+ #
199
+ # @param description [String] New description for the workflow.
200
+ #
201
+ # @param file_search [Array<String>] An array of file IDs to search in the workflow's evaluations. Files must be uplo
202
+ #
203
+ # @param improvement_action [Symbol, Deeprails::Models::DefendUpdateWorkflowParams::ImprovementAction] The new action used to improve outputs that fail one or more guardrail metrics f
204
+ #
205
+ # @param max_improvement_attempts [Integer] Max. number of improvement action attempts until a given event passes the guardr
206
+ #
207
+ # @param name [String] New name for the workflow.
208
+ #
209
+ # @param threshold_type [Symbol, Deeprails::Models::DefendUpdateWorkflowParams::ThresholdType] New type of thresholds to use for the workflow, either `automatic` or `custom`.
210
+ #
211
+ # @param web_search [Boolean] Whether to enable web search for this workflow's evaluations.
147
212
  #
148
213
  # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
149
214
  #