deeprails 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e97d49d00ebc0ed03f3be4d2e391e5091fdaf9cbb1fcd743e5800844c8f233c3
4
- data.tar.gz: 98898cc7823efe565ccd8f7519318d9af31fd9f3197c870e467c53cdc67ceaea
3
+ metadata.gz: 27710c0ec0510319e5002b296aa2eecb635640fee1a38423cf43e8792abd1c9a
4
+ data.tar.gz: 372425505cca1a9bfea94c6adbe93c99cf13cfdf26ea4506ae934e0049486acd
5
5
  SHA512:
6
- metadata.gz: 13b913ddb0433f6de13a5ea8a990b1ef1de1e308012fed87ffde7f42a21596508a280f56401e159a9c8a1a86b9b31d9825a13d35cd4d08486666bb4c57f434a7
7
- data.tar.gz: 1326ee104ab1202895c1d3eb667be76c933584fad15979b01505ad6bdf06c07a433762d986e5a9a51fc928070d59f196f3c9bb52d36ae34d27c231a20c0a055e
6
+ metadata.gz: 03e5bbb5226ef22805593349a2ccd1e4f888405bcb15b19eadc01fcd0e001d88a1b096ccfabc467f8b5cd6de6217d5dfc13122911a41b9866fb4467282e9d14f
7
+ data.tar.gz: 2bfd88ef33139e1ac30af2d3d4098e55061f6712736e61598908e972612cf1053eff0ba1cacc052c2a3e8522742617d320ae1fc471000ef9fc4f4749ad121fb8
data/CHANGELOG.md CHANGED
@@ -1,5 +1,13 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.11.0 (2025-11-02)
4
+
5
+ Full Changelog: [v0.10.0...v0.11.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.10.0...v0.11.0)
6
+
7
+ ### Features
8
+
9
+ * **api:** restructure monitor service ([9682519](https://github.com/deeprails/deeprails-ruby-sdk/commit/968251927003819495ee8c2e5ec9c9276f0a298d))
10
+
3
11
  ## 0.10.0 (2025-10-30)
4
12
 
5
13
  Full Changelog: [v0.9.0...v0.10.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.9.0...v0.10.0)
data/README.md CHANGED
@@ -15,7 +15,7 @@ To use this gem, install via Bundler by adding the following to your application
15
15
  <!-- x-release-please-start-version -->
16
16
 
17
17
  ```ruby
18
- gem "deeprails", "~> 0.10.0"
18
+ gem "deeprails", "~> 0.11.0"
19
19
  ```
20
20
 
21
21
  <!-- x-release-please-end -->
@@ -7,6 +7,16 @@ module Deeprails
7
7
  extend Deeprails::Internal::Type::RequestParameters::Converter
8
8
  include Deeprails::Internal::Type::RequestParameters
9
9
 
10
+ # @!attribute guardrail_metrics
11
+ # An array of guardrail metrics that the model input and output pair will be
12
+ # evaluated on. For non-enterprise users, these will be limited to `correctness`,
13
+ # `completeness`, `instruction_adherence`, `context_adherence`,
14
+ # `ground_truth_adherence`, and/or `comprehensive_safety`.
15
+ #
16
+ # @return [Array<Symbol, Deeprails::Models::MonitorCreateParams::GuardrailMetric>]
17
+ required :guardrail_metrics,
18
+ -> { Deeprails::Internal::Type::ArrayOf[enum: Deeprails::MonitorCreateParams::GuardrailMetric] }
19
+
10
20
  # @!attribute name
11
21
  # Name of the new monitor.
12
22
  #
@@ -19,12 +29,48 @@ module Deeprails
19
29
  # @return [String, nil]
20
30
  optional :description, String
21
31
 
22
- # @!method initialize(name:, description: nil, request_options: {})
32
+ # @!attribute file_search
33
+ # An array of file IDs to search in the monitor's evaluations. Files must be
34
+ # uploaded via the DeepRails API first.
35
+ #
36
+ # @return [Array<String>, nil]
37
+ optional :file_search, Deeprails::Internal::Type::ArrayOf[String]
38
+
39
+ # @!attribute web_search
40
+ # Whether to enable web search for this monitor's evaluations. Defaults to false.
41
+ #
42
+ # @return [Boolean, nil]
43
+ optional :web_search, Deeprails::Internal::Type::Boolean
44
+
45
+ # @!method initialize(guardrail_metrics:, name:, description: nil, file_search: nil, web_search: nil, request_options: {})
46
+ # Some parameter documentations has been truncated, see
47
+ # {Deeprails::Models::MonitorCreateParams} for more details.
48
+ #
49
+ # @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorCreateParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
50
+ #
23
51
  # @param name [String] Name of the new monitor.
24
52
  #
25
53
  # @param description [String] Description of the new monitor.
26
54
  #
55
+ # @param file_search [Array<String>] An array of file IDs to search in the monitor's evaluations. Files must be uploa
56
+ #
57
+ # @param web_search [Boolean] Whether to enable web search for this monitor's evaluations. Defaults to false.
58
+ #
27
59
  # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
60
+
61
+ module GuardrailMetric
62
+ extend Deeprails::Internal::Type::Enum
63
+
64
+ CORRECTNESS = :correctness
65
+ COMPLETENESS = :completeness
66
+ INSTRUCTION_ADHERENCE = :instruction_adherence
67
+ CONTEXT_ADHERENCE = :context_adherence
68
+ GROUND_TRUTH_ADHERENCE = :ground_truth_adherence
69
+ COMPREHENSIVE_SAFETY = :comprehensive_safety
70
+
71
+ # @!method self.values
72
+ # @return [Array<Symbol>]
73
+ end
28
74
  end
29
75
  end
30
76
  end
@@ -7,16 +7,6 @@ module Deeprails
7
7
  extend Deeprails::Internal::Type::RequestParameters::Converter
8
8
  include Deeprails::Internal::Type::RequestParameters
9
9
 
10
- # @!attribute guardrail_metrics
11
- # An array of guardrail metrics that the model input and output pair will be
12
- # evaluated on. For non-enterprise users, these will be limited to `correctness`,
13
- # `completeness`, `instruction_adherence`, `context_adherence`,
14
- # `ground_truth_adherence`, and/or `comprehensive_safety`.
15
- #
16
- # @return [Array<Symbol, Deeprails::Models::MonitorSubmitEventParams::GuardrailMetric>]
17
- required :guardrail_metrics,
18
- -> { Deeprails::Internal::Type::ArrayOf[enum: Deeprails::MonitorSubmitEventParams::GuardrailMetric] }
19
-
20
10
  # @!attribute model_input
21
11
  # A dictionary of inputs sent to the LLM to generate output. The dictionary must
22
12
  # contain at least a `user_prompt` field or a `system_prompt` field. For
@@ -31,12 +21,6 @@ module Deeprails
31
21
  # @return [String]
32
22
  required :model_output, String
33
23
 
34
- # @!attribute model_used
35
- # Model ID used to generate the output, like `gpt-4o` or `o3`.
36
- #
37
- # @return [String, nil]
38
- optional :model_used, String
39
-
40
24
  # @!attribute nametag
41
25
  # An optional, user-defined tag for the event.
42
26
  #
@@ -52,38 +36,20 @@ module Deeprails
52
36
  # @return [Symbol, Deeprails::Models::MonitorSubmitEventParams::RunMode, nil]
53
37
  optional :run_mode, enum: -> { Deeprails::MonitorSubmitEventParams::RunMode }
54
38
 
55
- # @!method initialize(guardrail_metrics:, model_input:, model_output:, model_used: nil, nametag: nil, run_mode: nil, request_options: {})
39
+ # @!method initialize(model_input:, model_output:, nametag: nil, run_mode: nil, request_options: {})
56
40
  # Some parameter documentations has been truncated, see
57
41
  # {Deeprails::Models::MonitorSubmitEventParams} for more details.
58
42
  #
59
- # @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorSubmitEventParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
60
- #
61
43
  # @param model_input [Deeprails::Models::MonitorSubmitEventParams::ModelInput] A dictionary of inputs sent to the LLM to generate output. The dictionary must c
62
44
  #
63
45
  # @param model_output [String] Output generated by the LLM to be evaluated.
64
46
  #
65
- # @param model_used [String] Model ID used to generate the output, like `gpt-4o` or `o3`.
66
- #
67
47
  # @param nametag [String] An optional, user-defined tag for the event.
68
48
  #
69
49
  # @param run_mode [Symbol, Deeprails::Models::MonitorSubmitEventParams::RunMode] Run mode for the monitor event. The run mode allows the user to optimize for sp
70
50
  #
71
51
  # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
72
52
 
73
- module GuardrailMetric
74
- extend Deeprails::Internal::Type::Enum
75
-
76
- CORRECTNESS = :correctness
77
- COMPLETENESS = :completeness
78
- INSTRUCTION_ADHERENCE = :instruction_adherence
79
- CONTEXT_ADHERENCE = :context_adherence
80
- GROUND_TRUTH_ADHERENCE = :ground_truth_adherence
81
- COMPREHENSIVE_SAFETY = :comprehensive_safety
82
-
83
- # @!method self.values
84
- # @return [Array<Symbol>]
85
- end
86
-
87
53
  class ModelInput < Deeprails::Internal::Type::BaseModel
88
54
  # @!attribute ground_truth
89
55
  # The ground truth for evaluating Ground Truth Adherence guardrail.
@@ -3,15 +3,24 @@
3
3
  module Deeprails
4
4
  module Resources
5
5
  class Monitor
6
+ # Some parameter documentations has been truncated, see
7
+ # {Deeprails::Models::MonitorCreateParams} for more details.
8
+ #
6
9
  # Use this endpoint to create a new monitor to evaluate model inputs and outputs
7
10
  # using guardrails
8
11
  #
9
- # @overload create(name:, description: nil, request_options: {})
12
+ # @overload create(guardrail_metrics:, name:, description: nil, file_search: nil, web_search: nil, request_options: {})
13
+ #
14
+ # @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorCreateParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
10
15
  #
11
16
  # @param name [String] Name of the new monitor.
12
17
  #
13
18
  # @param description [String] Description of the new monitor.
14
19
  #
20
+ # @param file_search [Array<String>] An array of file IDs to search in the monitor's evaluations. Files must be uploa
21
+ #
22
+ # @param web_search [Boolean] Whether to enable web search for this monitor's evaluations. Defaults to false.
23
+ #
15
24
  # @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
16
25
  #
17
26
  # @return [Deeprails::Models::MonitorResponse]
@@ -91,18 +100,14 @@ module Deeprails
91
100
  # Use this endpoint to submit a model input and output pair to a monitor for
92
101
  # evaluation
93
102
  #
94
- # @overload submit_event(monitor_id, guardrail_metrics:, model_input:, model_output:, model_used: nil, nametag: nil, run_mode: nil, request_options: {})
103
+ # @overload submit_event(monitor_id, model_input:, model_output:, nametag: nil, run_mode: nil, request_options: {})
95
104
  #
96
105
  # @param monitor_id [String] The ID of the monitor associated with this event.
97
106
  #
98
- # @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorSubmitEventParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
99
- #
100
107
  # @param model_input [Deeprails::Models::MonitorSubmitEventParams::ModelInput] A dictionary of inputs sent to the LLM to generate output. The dictionary must c
101
108
  #
102
109
  # @param model_output [String] Output generated by the LLM to be evaluated.
103
110
  #
104
- # @param model_used [String] Model ID used to generate the output, like `gpt-4o` or `o3`.
105
- #
106
111
  # @param nametag [String] An optional, user-defined tag for the event.
107
112
  #
108
113
  # @param run_mode [Symbol, Deeprails::Models::MonitorSubmitEventParams::RunMode] Run mode for the monitor event. The run mode allows the user to optimize for sp
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Deeprails
4
- VERSION = "0.10.0"
4
+ VERSION = "0.11.0"
5
5
  end
@@ -11,6 +11,17 @@ module Deeprails
11
11
  T.any(Deeprails::MonitorCreateParams, Deeprails::Internal::AnyHash)
12
12
  end
13
13
 
14
+ # An array of guardrail metrics that the model input and output pair will be
15
+ # evaluated on. For non-enterprise users, these will be limited to `correctness`,
16
+ # `completeness`, `instruction_adherence`, `context_adherence`,
17
+ # `ground_truth_adherence`, and/or `comprehensive_safety`.
18
+ sig do
19
+ returns(
20
+ T::Array[Deeprails::MonitorCreateParams::GuardrailMetric::OrSymbol]
21
+ )
22
+ end
23
+ attr_accessor :guardrail_metrics
24
+
14
25
  # Name of the new monitor.
15
26
  sig { returns(String) }
16
27
  attr_accessor :name
@@ -22,18 +33,47 @@ module Deeprails
22
33
  sig { params(description: String).void }
23
34
  attr_writer :description
24
35
 
36
+ # An array of file IDs to search in the monitor's evaluations. Files must be
37
+ # uploaded via the DeepRails API first.
38
+ sig { returns(T.nilable(T::Array[String])) }
39
+ attr_reader :file_search
40
+
41
+ sig { params(file_search: T::Array[String]).void }
42
+ attr_writer :file_search
43
+
44
+ # Whether to enable web search for this monitor's evaluations. Defaults to false.
45
+ sig { returns(T.nilable(T::Boolean)) }
46
+ attr_reader :web_search
47
+
48
+ sig { params(web_search: T::Boolean).void }
49
+ attr_writer :web_search
50
+
25
51
  sig do
26
52
  params(
53
+ guardrail_metrics:
54
+ T::Array[Deeprails::MonitorCreateParams::GuardrailMetric::OrSymbol],
27
55
  name: String,
28
56
  description: String,
57
+ file_search: T::Array[String],
58
+ web_search: T::Boolean,
29
59
  request_options: Deeprails::RequestOptions::OrHash
30
60
  ).returns(T.attached_class)
31
61
  end
32
62
  def self.new(
63
+ # An array of guardrail metrics that the model input and output pair will be
64
+ # evaluated on. For non-enterprise users, these will be limited to `correctness`,
65
+ # `completeness`, `instruction_adherence`, `context_adherence`,
66
+ # `ground_truth_adherence`, and/or `comprehensive_safety`.
67
+ guardrail_metrics:,
33
68
  # Name of the new monitor.
34
69
  name:,
35
70
  # Description of the new monitor.
36
71
  description: nil,
72
+ # An array of file IDs to search in the monitor's evaluations. Files must be
73
+ # uploaded via the DeepRails API first.
74
+ file_search: nil,
75
+ # Whether to enable web search for this monitor's evaluations. Defaults to false.
76
+ web_search: nil,
37
77
  request_options: {}
38
78
  )
39
79
  end
@@ -41,14 +81,71 @@ module Deeprails
41
81
  sig do
42
82
  override.returns(
43
83
  {
84
+ guardrail_metrics:
85
+ T::Array[
86
+ Deeprails::MonitorCreateParams::GuardrailMetric::OrSymbol
87
+ ],
44
88
  name: String,
45
89
  description: String,
90
+ file_search: T::Array[String],
91
+ web_search: T::Boolean,
46
92
  request_options: Deeprails::RequestOptions
47
93
  }
48
94
  )
49
95
  end
50
96
  def to_hash
51
97
  end
98
+
99
+ module GuardrailMetric
100
+ extend Deeprails::Internal::Type::Enum
101
+
102
+ TaggedSymbol =
103
+ T.type_alias do
104
+ T.all(Symbol, Deeprails::MonitorCreateParams::GuardrailMetric)
105
+ end
106
+ OrSymbol = T.type_alias { T.any(Symbol, String) }
107
+
108
+ CORRECTNESS =
109
+ T.let(
110
+ :correctness,
111
+ Deeprails::MonitorCreateParams::GuardrailMetric::TaggedSymbol
112
+ )
113
+ COMPLETENESS =
114
+ T.let(
115
+ :completeness,
116
+ Deeprails::MonitorCreateParams::GuardrailMetric::TaggedSymbol
117
+ )
118
+ INSTRUCTION_ADHERENCE =
119
+ T.let(
120
+ :instruction_adherence,
121
+ Deeprails::MonitorCreateParams::GuardrailMetric::TaggedSymbol
122
+ )
123
+ CONTEXT_ADHERENCE =
124
+ T.let(
125
+ :context_adherence,
126
+ Deeprails::MonitorCreateParams::GuardrailMetric::TaggedSymbol
127
+ )
128
+ GROUND_TRUTH_ADHERENCE =
129
+ T.let(
130
+ :ground_truth_adherence,
131
+ Deeprails::MonitorCreateParams::GuardrailMetric::TaggedSymbol
132
+ )
133
+ COMPREHENSIVE_SAFETY =
134
+ T.let(
135
+ :comprehensive_safety,
136
+ Deeprails::MonitorCreateParams::GuardrailMetric::TaggedSymbol
137
+ )
138
+
139
+ sig do
140
+ override.returns(
141
+ T::Array[
142
+ Deeprails::MonitorCreateParams::GuardrailMetric::TaggedSymbol
143
+ ]
144
+ )
145
+ end
146
+ def self.values
147
+ end
148
+ end
52
149
  end
53
150
  end
54
151
  end
@@ -14,19 +14,6 @@ module Deeprails
14
14
  )
15
15
  end
16
16
 
17
- # An array of guardrail metrics that the model input and output pair will be
18
- # evaluated on. For non-enterprise users, these will be limited to `correctness`,
19
- # `completeness`, `instruction_adherence`, `context_adherence`,
20
- # `ground_truth_adherence`, and/or `comprehensive_safety`.
21
- sig do
22
- returns(
23
- T::Array[
24
- Deeprails::MonitorSubmitEventParams::GuardrailMetric::OrSymbol
25
- ]
26
- )
27
- end
28
- attr_accessor :guardrail_metrics
29
-
30
17
  # A dictionary of inputs sent to the LLM to generate output. The dictionary must
31
18
  # contain at least a `user_prompt` field or a `system_prompt` field. For
32
19
  # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
@@ -44,13 +31,6 @@ module Deeprails
44
31
  sig { returns(String) }
45
32
  attr_accessor :model_output
46
33
 
47
- # Model ID used to generate the output, like `gpt-4o` or `o3`.
48
- sig { returns(T.nilable(String)) }
49
- attr_reader :model_used
50
-
51
- sig { params(model_used: String).void }
52
- attr_writer :model_used
53
-
54
34
  # An optional, user-defined tag for the event.
55
35
  sig { returns(T.nilable(String)) }
56
36
  attr_reader :nametag
@@ -78,32 +58,20 @@ module Deeprails
78
58
 
79
59
  sig do
80
60
  params(
81
- guardrail_metrics:
82
- T::Array[
83
- Deeprails::MonitorSubmitEventParams::GuardrailMetric::OrSymbol
84
- ],
85
61
  model_input: Deeprails::MonitorSubmitEventParams::ModelInput::OrHash,
86
62
  model_output: String,
87
- model_used: String,
88
63
  nametag: String,
89
64
  run_mode: Deeprails::MonitorSubmitEventParams::RunMode::OrSymbol,
90
65
  request_options: Deeprails::RequestOptions::OrHash
91
66
  ).returns(T.attached_class)
92
67
  end
93
68
  def self.new(
94
- # An array of guardrail metrics that the model input and output pair will be
95
- # evaluated on. For non-enterprise users, these will be limited to `correctness`,
96
- # `completeness`, `instruction_adherence`, `context_adherence`,
97
- # `ground_truth_adherence`, and/or `comprehensive_safety`.
98
- guardrail_metrics:,
99
69
  # A dictionary of inputs sent to the LLM to generate output. The dictionary must
100
70
  # contain at least a `user_prompt` field or a `system_prompt` field. For
101
71
  # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
102
72
  model_input:,
103
73
  # Output generated by the LLM to be evaluated.
104
74
  model_output:,
105
- # Model ID used to generate the output, like `gpt-4o` or `o3`.
106
- model_used: nil,
107
75
  # An optional, user-defined tag for the event.
108
76
  nametag: nil,
109
77
  # Run mode for the monitor event. The run mode allows the user to optimize for
@@ -118,13 +86,8 @@ module Deeprails
118
86
  sig do
119
87
  override.returns(
120
88
  {
121
- guardrail_metrics:
122
- T::Array[
123
- Deeprails::MonitorSubmitEventParams::GuardrailMetric::OrSymbol
124
- ],
125
89
  model_input: Deeprails::MonitorSubmitEventParams::ModelInput,
126
90
  model_output: String,
127
- model_used: String,
128
91
  nametag: String,
129
92
  run_mode: Deeprails::MonitorSubmitEventParams::RunMode::OrSymbol,
130
93
  request_options: Deeprails::RequestOptions
@@ -134,57 +97,6 @@ module Deeprails
134
97
  def to_hash
135
98
  end
136
99
 
137
- module GuardrailMetric
138
- extend Deeprails::Internal::Type::Enum
139
-
140
- TaggedSymbol =
141
- T.type_alias do
142
- T.all(Symbol, Deeprails::MonitorSubmitEventParams::GuardrailMetric)
143
- end
144
- OrSymbol = T.type_alias { T.any(Symbol, String) }
145
-
146
- CORRECTNESS =
147
- T.let(
148
- :correctness,
149
- Deeprails::MonitorSubmitEventParams::GuardrailMetric::TaggedSymbol
150
- )
151
- COMPLETENESS =
152
- T.let(
153
- :completeness,
154
- Deeprails::MonitorSubmitEventParams::GuardrailMetric::TaggedSymbol
155
- )
156
- INSTRUCTION_ADHERENCE =
157
- T.let(
158
- :instruction_adherence,
159
- Deeprails::MonitorSubmitEventParams::GuardrailMetric::TaggedSymbol
160
- )
161
- CONTEXT_ADHERENCE =
162
- T.let(
163
- :context_adherence,
164
- Deeprails::MonitorSubmitEventParams::GuardrailMetric::TaggedSymbol
165
- )
166
- GROUND_TRUTH_ADHERENCE =
167
- T.let(
168
- :ground_truth_adherence,
169
- Deeprails::MonitorSubmitEventParams::GuardrailMetric::TaggedSymbol
170
- )
171
- COMPREHENSIVE_SAFETY =
172
- T.let(
173
- :comprehensive_safety,
174
- Deeprails::MonitorSubmitEventParams::GuardrailMetric::TaggedSymbol
175
- )
176
-
177
- sig do
178
- override.returns(
179
- T::Array[
180
- Deeprails::MonitorSubmitEventParams::GuardrailMetric::TaggedSymbol
181
- ]
182
- )
183
- end
184
- def self.values
185
- end
186
- end
187
-
188
100
  class ModelInput < Deeprails::Internal::Type::BaseModel
189
101
  OrHash =
190
102
  T.type_alias do
@@ -7,16 +7,30 @@ module Deeprails
7
7
  # using guardrails
8
8
  sig do
9
9
  params(
10
+ guardrail_metrics:
11
+ T::Array[Deeprails::MonitorCreateParams::GuardrailMetric::OrSymbol],
10
12
  name: String,
11
13
  description: String,
14
+ file_search: T::Array[String],
15
+ web_search: T::Boolean,
12
16
  request_options: Deeprails::RequestOptions::OrHash
13
17
  ).returns(Deeprails::MonitorResponse)
14
18
  end
15
19
  def create(
20
+ # An array of guardrail metrics that the model input and output pair will be
21
+ # evaluated on. For non-enterprise users, these will be limited to `correctness`,
22
+ # `completeness`, `instruction_adherence`, `context_adherence`,
23
+ # `ground_truth_adherence`, and/or `comprehensive_safety`.
24
+ guardrail_metrics:,
16
25
  # Name of the new monitor.
17
26
  name:,
18
27
  # Description of the new monitor.
19
28
  description: nil,
29
+ # An array of file IDs to search in the monitor's evaluations. Files must be
30
+ # uploaded via the DeepRails API first.
31
+ file_search: nil,
32
+ # Whether to enable web search for this monitor's evaluations. Defaults to false.
33
+ web_search: nil,
20
34
  request_options: {}
21
35
  )
22
36
  end
@@ -70,13 +84,8 @@ module Deeprails
70
84
  sig do
71
85
  params(
72
86
  monitor_id: String,
73
- guardrail_metrics:
74
- T::Array[
75
- Deeprails::MonitorSubmitEventParams::GuardrailMetric::OrSymbol
76
- ],
77
87
  model_input: Deeprails::MonitorSubmitEventParams::ModelInput::OrHash,
78
88
  model_output: String,
79
- model_used: String,
80
89
  nametag: String,
81
90
  run_mode: Deeprails::MonitorSubmitEventParams::RunMode::OrSymbol,
82
91
  request_options: Deeprails::RequestOptions::OrHash
@@ -85,19 +94,12 @@ module Deeprails
85
94
  def submit_event(
86
95
  # The ID of the monitor associated with this event.
87
96
  monitor_id,
88
- # An array of guardrail metrics that the model input and output pair will be
89
- # evaluated on. For non-enterprise users, these will be limited to `correctness`,
90
- # `completeness`, `instruction_adherence`, `context_adherence`,
91
- # `ground_truth_adherence`, and/or `comprehensive_safety`.
92
- guardrail_metrics:,
93
97
  # A dictionary of inputs sent to the LLM to generate output. The dictionary must
94
98
  # contain at least a `user_prompt` field or a `system_prompt` field. For
95
99
  # ground_truth_adherence guardrail metric, `ground_truth` should be provided.
96
100
  model_input:,
97
101
  # Output generated by the LLM to be evaluated.
98
102
  model_output:,
99
- # Model ID used to generate the output, like `gpt-4o` or `o3`.
100
- model_used: nil,
101
103
  # An optional, user-defined tag for the event.
102
104
  nametag: nil,
103
105
  # Run mode for the monitor event. The run mode allows the user to optimize for
@@ -1,30 +1,73 @@
1
1
  module Deeprails
2
2
  module Models
3
3
  type monitor_create_params =
4
- { name: String, description: String }
4
+ {
5
+ guardrail_metrics: ::Array[Deeprails::Models::MonitorCreateParams::guardrail_metric],
6
+ name: String,
7
+ description: String,
8
+ file_search: ::Array[String],
9
+ web_search: bool
10
+ }
5
11
  & Deeprails::Internal::Type::request_parameters
6
12
 
7
13
  class MonitorCreateParams < Deeprails::Internal::Type::BaseModel
8
14
  extend Deeprails::Internal::Type::RequestParameters::Converter
9
15
  include Deeprails::Internal::Type::RequestParameters
10
16
 
17
+ attr_accessor guardrail_metrics: ::Array[Deeprails::Models::MonitorCreateParams::guardrail_metric]
18
+
11
19
  attr_accessor name: String
12
20
 
13
21
  attr_reader description: String?
14
22
 
15
23
  def description=: (String) -> String
16
24
 
25
+ attr_reader file_search: ::Array[String]?
26
+
27
+ def file_search=: (::Array[String]) -> ::Array[String]
28
+
29
+ attr_reader web_search: bool?
30
+
31
+ def web_search=: (bool) -> bool
32
+
17
33
  def initialize: (
34
+ guardrail_metrics: ::Array[Deeprails::Models::MonitorCreateParams::guardrail_metric],
18
35
  name: String,
19
36
  ?description: String,
37
+ ?file_search: ::Array[String],
38
+ ?web_search: bool,
20
39
  ?request_options: Deeprails::request_opts
21
40
  ) -> void
22
41
 
23
42
  def to_hash: -> {
43
+ guardrail_metrics: ::Array[Deeprails::Models::MonitorCreateParams::guardrail_metric],
24
44
  name: String,
25
45
  description: String,
46
+ file_search: ::Array[String],
47
+ web_search: bool,
26
48
  request_options: Deeprails::RequestOptions
27
49
  }
50
+
51
+ type guardrail_metric =
52
+ :correctness
53
+ | :completeness
54
+ | :instruction_adherence
55
+ | :context_adherence
56
+ | :ground_truth_adherence
57
+ | :comprehensive_safety
58
+
59
+ module GuardrailMetric
60
+ extend Deeprails::Internal::Type::Enum
61
+
62
+ CORRECTNESS: :correctness
63
+ COMPLETENESS: :completeness
64
+ INSTRUCTION_ADHERENCE: :instruction_adherence
65
+ CONTEXT_ADHERENCE: :context_adherence
66
+ GROUND_TRUTH_ADHERENCE: :ground_truth_adherence
67
+ COMPREHENSIVE_SAFETY: :comprehensive_safety
68
+
69
+ def self?.values: -> ::Array[Deeprails::Models::MonitorCreateParams::guardrail_metric]
70
+ end
28
71
  end
29
72
  end
30
73
  end
@@ -2,10 +2,8 @@ module Deeprails
2
2
  module Models
3
3
  type monitor_submit_event_params =
4
4
  {
5
- guardrail_metrics: ::Array[Deeprails::Models::MonitorSubmitEventParams::guardrail_metric],
6
5
  model_input: Deeprails::MonitorSubmitEventParams::ModelInput,
7
6
  model_output: String,
8
- model_used: String,
9
7
  nametag: String,
10
8
  run_mode: Deeprails::Models::MonitorSubmitEventParams::run_mode
11
9
  }
@@ -15,16 +13,10 @@ module Deeprails
15
13
  extend Deeprails::Internal::Type::RequestParameters::Converter
16
14
  include Deeprails::Internal::Type::RequestParameters
17
15
 
18
- attr_accessor guardrail_metrics: ::Array[Deeprails::Models::MonitorSubmitEventParams::guardrail_metric]
19
-
20
16
  attr_accessor model_input: Deeprails::MonitorSubmitEventParams::ModelInput
21
17
 
22
18
  attr_accessor model_output: String
23
19
 
24
- attr_reader model_used: String?
25
-
26
- def model_used=: (String) -> String
27
-
28
20
  attr_reader nametag: String?
29
21
 
30
22
  def nametag=: (String) -> String
@@ -36,46 +28,21 @@ module Deeprails
36
28
  ) -> Deeprails::Models::MonitorSubmitEventParams::run_mode
37
29
 
38
30
  def initialize: (
39
- guardrail_metrics: ::Array[Deeprails::Models::MonitorSubmitEventParams::guardrail_metric],
40
31
  model_input: Deeprails::MonitorSubmitEventParams::ModelInput,
41
32
  model_output: String,
42
- ?model_used: String,
43
33
  ?nametag: String,
44
34
  ?run_mode: Deeprails::Models::MonitorSubmitEventParams::run_mode,
45
35
  ?request_options: Deeprails::request_opts
46
36
  ) -> void
47
37
 
48
38
  def to_hash: -> {
49
- guardrail_metrics: ::Array[Deeprails::Models::MonitorSubmitEventParams::guardrail_metric],
50
39
  model_input: Deeprails::MonitorSubmitEventParams::ModelInput,
51
40
  model_output: String,
52
- model_used: String,
53
41
  nametag: String,
54
42
  run_mode: Deeprails::Models::MonitorSubmitEventParams::run_mode,
55
43
  request_options: Deeprails::RequestOptions
56
44
  }
57
45
 
58
- type guardrail_metric =
59
- :correctness
60
- | :completeness
61
- | :instruction_adherence
62
- | :context_adherence
63
- | :ground_truth_adherence
64
- | :comprehensive_safety
65
-
66
- module GuardrailMetric
67
- extend Deeprails::Internal::Type::Enum
68
-
69
- CORRECTNESS: :correctness
70
- COMPLETENESS: :completeness
71
- INSTRUCTION_ADHERENCE: :instruction_adherence
72
- CONTEXT_ADHERENCE: :context_adherence
73
- GROUND_TRUTH_ADHERENCE: :ground_truth_adherence
74
- COMPREHENSIVE_SAFETY: :comprehensive_safety
75
-
76
- def self?.values: -> ::Array[Deeprails::Models::MonitorSubmitEventParams::guardrail_metric]
77
- end
78
-
79
46
  type model_input =
80
47
  { ground_truth: String, system_prompt: String, user_prompt: String }
81
48
 
@@ -2,8 +2,11 @@ module Deeprails
2
2
  module Resources
3
3
  class Monitor
4
4
  def create: (
5
+ guardrail_metrics: ::Array[Deeprails::Models::MonitorCreateParams::guardrail_metric],
5
6
  name: String,
6
7
  ?description: String,
8
+ ?file_search: ::Array[String],
9
+ ?web_search: bool,
7
10
  ?request_options: Deeprails::request_opts
8
11
  ) -> Deeprails::MonitorResponse
9
12
 
@@ -23,10 +26,8 @@ module Deeprails
23
26
 
24
27
  def submit_event: (
25
28
  String monitor_id,
26
- guardrail_metrics: ::Array[Deeprails::Models::MonitorSubmitEventParams::guardrail_metric],
27
29
  model_input: Deeprails::MonitorSubmitEventParams::ModelInput,
28
30
  model_output: String,
29
- ?model_used: String,
30
31
  ?nametag: String,
31
32
  ?run_mode: Deeprails::Models::MonitorSubmitEventParams::run_mode,
32
33
  ?request_options: Deeprails::request_opts
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: deeprails
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.10.0
4
+ version: 0.11.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Deeprails
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2025-10-30 00:00:00.000000000 Z
11
+ date: 2025-11-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: connection_pool