deeprails 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/README.md +1 -1
- data/lib/deeprails/models/monitor_create_params.rb +47 -1
- data/lib/deeprails/models/monitor_submit_event_params.rb +1 -35
- data/lib/deeprails/resources/monitor.rb +11 -6
- data/lib/deeprails/version.rb +1 -1
- data/rbi/deeprails/models/monitor_create_params.rbi +97 -0
- data/rbi/deeprails/models/monitor_submit_event_params.rbi +0 -88
- data/rbi/deeprails/resources/monitor.rbi +14 -12
- data/sig/deeprails/models/monitor_create_params.rbs +44 -1
- data/sig/deeprails/models/monitor_submit_event_params.rbs +0 -33
- data/sig/deeprails/resources/monitor.rbs +3 -2
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 27710c0ec0510319e5002b296aa2eecb635640fee1a38423cf43e8792abd1c9a
|
|
4
|
+
data.tar.gz: 372425505cca1a9bfea94c6adbe93c99cf13cfdf26ea4506ae934e0049486acd
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 03e5bbb5226ef22805593349a2ccd1e4f888405bcb15b19eadc01fcd0e001d88a1b096ccfabc467f8b5cd6de6217d5dfc13122911a41b9866fb4467282e9d14f
|
|
7
|
+
data.tar.gz: 2bfd88ef33139e1ac30af2d3d4098e55061f6712736e61598908e972612cf1053eff0ba1cacc052c2a3e8522742617d320ae1fc471000ef9fc4f4749ad121fb8
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.11.0 (2025-11-02)
|
|
4
|
+
|
|
5
|
+
Full Changelog: [v0.10.0...v0.11.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.10.0...v0.11.0)
|
|
6
|
+
|
|
7
|
+
### Features
|
|
8
|
+
|
|
9
|
+
* **api:** restructure monitor service ([9682519](https://github.com/deeprails/deeprails-ruby-sdk/commit/968251927003819495ee8c2e5ec9c9276f0a298d))
|
|
10
|
+
|
|
3
11
|
## 0.10.0 (2025-10-30)
|
|
4
12
|
|
|
5
13
|
Full Changelog: [v0.9.0...v0.10.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.9.0...v0.10.0)
|
data/README.md
CHANGED
|
@@ -7,6 +7,16 @@ module Deeprails
|
|
|
7
7
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
8
8
|
include Deeprails::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
|
+
# @!attribute guardrail_metrics
|
|
11
|
+
# An array of guardrail metrics that the model input and output pair will be
|
|
12
|
+
# evaluated on. For non-enterprise users, these will be limited to `correctness`,
|
|
13
|
+
# `completeness`, `instruction_adherence`, `context_adherence`,
|
|
14
|
+
# `ground_truth_adherence`, and/or `comprehensive_safety`.
|
|
15
|
+
#
|
|
16
|
+
# @return [Array<Symbol, Deeprails::Models::MonitorCreateParams::GuardrailMetric>]
|
|
17
|
+
required :guardrail_metrics,
|
|
18
|
+
-> { Deeprails::Internal::Type::ArrayOf[enum: Deeprails::MonitorCreateParams::GuardrailMetric] }
|
|
19
|
+
|
|
10
20
|
# @!attribute name
|
|
11
21
|
# Name of the new monitor.
|
|
12
22
|
#
|
|
@@ -19,12 +29,48 @@ module Deeprails
|
|
|
19
29
|
# @return [String, nil]
|
|
20
30
|
optional :description, String
|
|
21
31
|
|
|
22
|
-
# @!
|
|
32
|
+
# @!attribute file_search
|
|
33
|
+
# An array of file IDs to search in the monitor's evaluations. Files must be
|
|
34
|
+
# uploaded via the DeepRails API first.
|
|
35
|
+
#
|
|
36
|
+
# @return [Array<String>, nil]
|
|
37
|
+
optional :file_search, Deeprails::Internal::Type::ArrayOf[String]
|
|
38
|
+
|
|
39
|
+
# @!attribute web_search
|
|
40
|
+
# Whether to enable web search for this monitor's evaluations. Defaults to false.
|
|
41
|
+
#
|
|
42
|
+
# @return [Boolean, nil]
|
|
43
|
+
optional :web_search, Deeprails::Internal::Type::Boolean
|
|
44
|
+
|
|
45
|
+
# @!method initialize(guardrail_metrics:, name:, description: nil, file_search: nil, web_search: nil, request_options: {})
|
|
46
|
+
# Some parameter documentations has been truncated, see
|
|
47
|
+
# {Deeprails::Models::MonitorCreateParams} for more details.
|
|
48
|
+
#
|
|
49
|
+
# @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorCreateParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
|
|
50
|
+
#
|
|
23
51
|
# @param name [String] Name of the new monitor.
|
|
24
52
|
#
|
|
25
53
|
# @param description [String] Description of the new monitor.
|
|
26
54
|
#
|
|
55
|
+
# @param file_search [Array<String>] An array of file IDs to search in the monitor's evaluations. Files must be uploa
|
|
56
|
+
#
|
|
57
|
+
# @param web_search [Boolean] Whether to enable web search for this monitor's evaluations. Defaults to false.
|
|
58
|
+
#
|
|
27
59
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
60
|
+
|
|
61
|
+
module GuardrailMetric
|
|
62
|
+
extend Deeprails::Internal::Type::Enum
|
|
63
|
+
|
|
64
|
+
CORRECTNESS = :correctness
|
|
65
|
+
COMPLETENESS = :completeness
|
|
66
|
+
INSTRUCTION_ADHERENCE = :instruction_adherence
|
|
67
|
+
CONTEXT_ADHERENCE = :context_adherence
|
|
68
|
+
GROUND_TRUTH_ADHERENCE = :ground_truth_adherence
|
|
69
|
+
COMPREHENSIVE_SAFETY = :comprehensive_safety
|
|
70
|
+
|
|
71
|
+
# @!method self.values
|
|
72
|
+
# @return [Array<Symbol>]
|
|
73
|
+
end
|
|
28
74
|
end
|
|
29
75
|
end
|
|
30
76
|
end
|
|
@@ -7,16 +7,6 @@ module Deeprails
|
|
|
7
7
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
8
8
|
include Deeprails::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
|
-
# @!attribute guardrail_metrics
|
|
11
|
-
# An array of guardrail metrics that the model input and output pair will be
|
|
12
|
-
# evaluated on. For non-enterprise users, these will be limited to `correctness`,
|
|
13
|
-
# `completeness`, `instruction_adherence`, `context_adherence`,
|
|
14
|
-
# `ground_truth_adherence`, and/or `comprehensive_safety`.
|
|
15
|
-
#
|
|
16
|
-
# @return [Array<Symbol, Deeprails::Models::MonitorSubmitEventParams::GuardrailMetric>]
|
|
17
|
-
required :guardrail_metrics,
|
|
18
|
-
-> { Deeprails::Internal::Type::ArrayOf[enum: Deeprails::MonitorSubmitEventParams::GuardrailMetric] }
|
|
19
|
-
|
|
20
10
|
# @!attribute model_input
|
|
21
11
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
22
12
|
# contain at least a `user_prompt` field or a `system_prompt` field. For
|
|
@@ -31,12 +21,6 @@ module Deeprails
|
|
|
31
21
|
# @return [String]
|
|
32
22
|
required :model_output, String
|
|
33
23
|
|
|
34
|
-
# @!attribute model_used
|
|
35
|
-
# Model ID used to generate the output, like `gpt-4o` or `o3`.
|
|
36
|
-
#
|
|
37
|
-
# @return [String, nil]
|
|
38
|
-
optional :model_used, String
|
|
39
|
-
|
|
40
24
|
# @!attribute nametag
|
|
41
25
|
# An optional, user-defined tag for the event.
|
|
42
26
|
#
|
|
@@ -52,38 +36,20 @@ module Deeprails
|
|
|
52
36
|
# @return [Symbol, Deeprails::Models::MonitorSubmitEventParams::RunMode, nil]
|
|
53
37
|
optional :run_mode, enum: -> { Deeprails::MonitorSubmitEventParams::RunMode }
|
|
54
38
|
|
|
55
|
-
# @!method initialize(
|
|
39
|
+
# @!method initialize(model_input:, model_output:, nametag: nil, run_mode: nil, request_options: {})
|
|
56
40
|
# Some parameter documentations has been truncated, see
|
|
57
41
|
# {Deeprails::Models::MonitorSubmitEventParams} for more details.
|
|
58
42
|
#
|
|
59
|
-
# @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorSubmitEventParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
|
|
60
|
-
#
|
|
61
43
|
# @param model_input [Deeprails::Models::MonitorSubmitEventParams::ModelInput] A dictionary of inputs sent to the LLM to generate output. The dictionary must c
|
|
62
44
|
#
|
|
63
45
|
# @param model_output [String] Output generated by the LLM to be evaluated.
|
|
64
46
|
#
|
|
65
|
-
# @param model_used [String] Model ID used to generate the output, like `gpt-4o` or `o3`.
|
|
66
|
-
#
|
|
67
47
|
# @param nametag [String] An optional, user-defined tag for the event.
|
|
68
48
|
#
|
|
69
49
|
# @param run_mode [Symbol, Deeprails::Models::MonitorSubmitEventParams::RunMode] Run mode for the monitor event. The run mode allows the user to optimize for sp
|
|
70
50
|
#
|
|
71
51
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
72
52
|
|
|
73
|
-
module GuardrailMetric
|
|
74
|
-
extend Deeprails::Internal::Type::Enum
|
|
75
|
-
|
|
76
|
-
CORRECTNESS = :correctness
|
|
77
|
-
COMPLETENESS = :completeness
|
|
78
|
-
INSTRUCTION_ADHERENCE = :instruction_adherence
|
|
79
|
-
CONTEXT_ADHERENCE = :context_adherence
|
|
80
|
-
GROUND_TRUTH_ADHERENCE = :ground_truth_adherence
|
|
81
|
-
COMPREHENSIVE_SAFETY = :comprehensive_safety
|
|
82
|
-
|
|
83
|
-
# @!method self.values
|
|
84
|
-
# @return [Array<Symbol>]
|
|
85
|
-
end
|
|
86
|
-
|
|
87
53
|
class ModelInput < Deeprails::Internal::Type::BaseModel
|
|
88
54
|
# @!attribute ground_truth
|
|
89
55
|
# The ground truth for evaluating Ground Truth Adherence guardrail.
|
|
@@ -3,15 +3,24 @@
|
|
|
3
3
|
module Deeprails
|
|
4
4
|
module Resources
|
|
5
5
|
class Monitor
|
|
6
|
+
# Some parameter documentations has been truncated, see
|
|
7
|
+
# {Deeprails::Models::MonitorCreateParams} for more details.
|
|
8
|
+
#
|
|
6
9
|
# Use this endpoint to create a new monitor to evaluate model inputs and outputs
|
|
7
10
|
# using guardrails
|
|
8
11
|
#
|
|
9
|
-
# @overload create(name:, description: nil, request_options: {})
|
|
12
|
+
# @overload create(guardrail_metrics:, name:, description: nil, file_search: nil, web_search: nil, request_options: {})
|
|
13
|
+
#
|
|
14
|
+
# @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorCreateParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
|
|
10
15
|
#
|
|
11
16
|
# @param name [String] Name of the new monitor.
|
|
12
17
|
#
|
|
13
18
|
# @param description [String] Description of the new monitor.
|
|
14
19
|
#
|
|
20
|
+
# @param file_search [Array<String>] An array of file IDs to search in the monitor's evaluations. Files must be uploa
|
|
21
|
+
#
|
|
22
|
+
# @param web_search [Boolean] Whether to enable web search for this monitor's evaluations. Defaults to false.
|
|
23
|
+
#
|
|
15
24
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
16
25
|
#
|
|
17
26
|
# @return [Deeprails::Models::MonitorResponse]
|
|
@@ -91,18 +100,14 @@ module Deeprails
|
|
|
91
100
|
# Use this endpoint to submit a model input and output pair to a monitor for
|
|
92
101
|
# evaluation
|
|
93
102
|
#
|
|
94
|
-
# @overload submit_event(monitor_id,
|
|
103
|
+
# @overload submit_event(monitor_id, model_input:, model_output:, nametag: nil, run_mode: nil, request_options: {})
|
|
95
104
|
#
|
|
96
105
|
# @param monitor_id [String] The ID of the monitor associated with this event.
|
|
97
106
|
#
|
|
98
|
-
# @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorSubmitEventParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
|
|
99
|
-
#
|
|
100
107
|
# @param model_input [Deeprails::Models::MonitorSubmitEventParams::ModelInput] A dictionary of inputs sent to the LLM to generate output. The dictionary must c
|
|
101
108
|
#
|
|
102
109
|
# @param model_output [String] Output generated by the LLM to be evaluated.
|
|
103
110
|
#
|
|
104
|
-
# @param model_used [String] Model ID used to generate the output, like `gpt-4o` or `o3`.
|
|
105
|
-
#
|
|
106
111
|
# @param nametag [String] An optional, user-defined tag for the event.
|
|
107
112
|
#
|
|
108
113
|
# @param run_mode [Symbol, Deeprails::Models::MonitorSubmitEventParams::RunMode] Run mode for the monitor event. The run mode allows the user to optimize for sp
|
data/lib/deeprails/version.rb
CHANGED
|
@@ -11,6 +11,17 @@ module Deeprails
|
|
|
11
11
|
T.any(Deeprails::MonitorCreateParams, Deeprails::Internal::AnyHash)
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
+
# An array of guardrail metrics that the model input and output pair will be
|
|
15
|
+
# evaluated on. For non-enterprise users, these will be limited to `correctness`,
|
|
16
|
+
# `completeness`, `instruction_adherence`, `context_adherence`,
|
|
17
|
+
# `ground_truth_adherence`, and/or `comprehensive_safety`.
|
|
18
|
+
sig do
|
|
19
|
+
returns(
|
|
20
|
+
T::Array[Deeprails::MonitorCreateParams::GuardrailMetric::OrSymbol]
|
|
21
|
+
)
|
|
22
|
+
end
|
|
23
|
+
attr_accessor :guardrail_metrics
|
|
24
|
+
|
|
14
25
|
# Name of the new monitor.
|
|
15
26
|
sig { returns(String) }
|
|
16
27
|
attr_accessor :name
|
|
@@ -22,18 +33,47 @@ module Deeprails
|
|
|
22
33
|
sig { params(description: String).void }
|
|
23
34
|
attr_writer :description
|
|
24
35
|
|
|
36
|
+
# An array of file IDs to search in the monitor's evaluations. Files must be
|
|
37
|
+
# uploaded via the DeepRails API first.
|
|
38
|
+
sig { returns(T.nilable(T::Array[String])) }
|
|
39
|
+
attr_reader :file_search
|
|
40
|
+
|
|
41
|
+
sig { params(file_search: T::Array[String]).void }
|
|
42
|
+
attr_writer :file_search
|
|
43
|
+
|
|
44
|
+
# Whether to enable web search for this monitor's evaluations. Defaults to false.
|
|
45
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
46
|
+
attr_reader :web_search
|
|
47
|
+
|
|
48
|
+
sig { params(web_search: T::Boolean).void }
|
|
49
|
+
attr_writer :web_search
|
|
50
|
+
|
|
25
51
|
sig do
|
|
26
52
|
params(
|
|
53
|
+
guardrail_metrics:
|
|
54
|
+
T::Array[Deeprails::MonitorCreateParams::GuardrailMetric::OrSymbol],
|
|
27
55
|
name: String,
|
|
28
56
|
description: String,
|
|
57
|
+
file_search: T::Array[String],
|
|
58
|
+
web_search: T::Boolean,
|
|
29
59
|
request_options: Deeprails::RequestOptions::OrHash
|
|
30
60
|
).returns(T.attached_class)
|
|
31
61
|
end
|
|
32
62
|
def self.new(
|
|
63
|
+
# An array of guardrail metrics that the model input and output pair will be
|
|
64
|
+
# evaluated on. For non-enterprise users, these will be limited to `correctness`,
|
|
65
|
+
# `completeness`, `instruction_adherence`, `context_adherence`,
|
|
66
|
+
# `ground_truth_adherence`, and/or `comprehensive_safety`.
|
|
67
|
+
guardrail_metrics:,
|
|
33
68
|
# Name of the new monitor.
|
|
34
69
|
name:,
|
|
35
70
|
# Description of the new monitor.
|
|
36
71
|
description: nil,
|
|
72
|
+
# An array of file IDs to search in the monitor's evaluations. Files must be
|
|
73
|
+
# uploaded via the DeepRails API first.
|
|
74
|
+
file_search: nil,
|
|
75
|
+
# Whether to enable web search for this monitor's evaluations. Defaults to false.
|
|
76
|
+
web_search: nil,
|
|
37
77
|
request_options: {}
|
|
38
78
|
)
|
|
39
79
|
end
|
|
@@ -41,14 +81,71 @@ module Deeprails
|
|
|
41
81
|
sig do
|
|
42
82
|
override.returns(
|
|
43
83
|
{
|
|
84
|
+
guardrail_metrics:
|
|
85
|
+
T::Array[
|
|
86
|
+
Deeprails::MonitorCreateParams::GuardrailMetric::OrSymbol
|
|
87
|
+
],
|
|
44
88
|
name: String,
|
|
45
89
|
description: String,
|
|
90
|
+
file_search: T::Array[String],
|
|
91
|
+
web_search: T::Boolean,
|
|
46
92
|
request_options: Deeprails::RequestOptions
|
|
47
93
|
}
|
|
48
94
|
)
|
|
49
95
|
end
|
|
50
96
|
def to_hash
|
|
51
97
|
end
|
|
98
|
+
|
|
99
|
+
module GuardrailMetric
|
|
100
|
+
extend Deeprails::Internal::Type::Enum
|
|
101
|
+
|
|
102
|
+
TaggedSymbol =
|
|
103
|
+
T.type_alias do
|
|
104
|
+
T.all(Symbol, Deeprails::MonitorCreateParams::GuardrailMetric)
|
|
105
|
+
end
|
|
106
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
107
|
+
|
|
108
|
+
CORRECTNESS =
|
|
109
|
+
T.let(
|
|
110
|
+
:correctness,
|
|
111
|
+
Deeprails::MonitorCreateParams::GuardrailMetric::TaggedSymbol
|
|
112
|
+
)
|
|
113
|
+
COMPLETENESS =
|
|
114
|
+
T.let(
|
|
115
|
+
:completeness,
|
|
116
|
+
Deeprails::MonitorCreateParams::GuardrailMetric::TaggedSymbol
|
|
117
|
+
)
|
|
118
|
+
INSTRUCTION_ADHERENCE =
|
|
119
|
+
T.let(
|
|
120
|
+
:instruction_adherence,
|
|
121
|
+
Deeprails::MonitorCreateParams::GuardrailMetric::TaggedSymbol
|
|
122
|
+
)
|
|
123
|
+
CONTEXT_ADHERENCE =
|
|
124
|
+
T.let(
|
|
125
|
+
:context_adherence,
|
|
126
|
+
Deeprails::MonitorCreateParams::GuardrailMetric::TaggedSymbol
|
|
127
|
+
)
|
|
128
|
+
GROUND_TRUTH_ADHERENCE =
|
|
129
|
+
T.let(
|
|
130
|
+
:ground_truth_adherence,
|
|
131
|
+
Deeprails::MonitorCreateParams::GuardrailMetric::TaggedSymbol
|
|
132
|
+
)
|
|
133
|
+
COMPREHENSIVE_SAFETY =
|
|
134
|
+
T.let(
|
|
135
|
+
:comprehensive_safety,
|
|
136
|
+
Deeprails::MonitorCreateParams::GuardrailMetric::TaggedSymbol
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
sig do
|
|
140
|
+
override.returns(
|
|
141
|
+
T::Array[
|
|
142
|
+
Deeprails::MonitorCreateParams::GuardrailMetric::TaggedSymbol
|
|
143
|
+
]
|
|
144
|
+
)
|
|
145
|
+
end
|
|
146
|
+
def self.values
|
|
147
|
+
end
|
|
148
|
+
end
|
|
52
149
|
end
|
|
53
150
|
end
|
|
54
151
|
end
|
|
@@ -14,19 +14,6 @@ module Deeprails
|
|
|
14
14
|
)
|
|
15
15
|
end
|
|
16
16
|
|
|
17
|
-
# An array of guardrail metrics that the model input and output pair will be
|
|
18
|
-
# evaluated on. For non-enterprise users, these will be limited to `correctness`,
|
|
19
|
-
# `completeness`, `instruction_adherence`, `context_adherence`,
|
|
20
|
-
# `ground_truth_adherence`, and/or `comprehensive_safety`.
|
|
21
|
-
sig do
|
|
22
|
-
returns(
|
|
23
|
-
T::Array[
|
|
24
|
-
Deeprails::MonitorSubmitEventParams::GuardrailMetric::OrSymbol
|
|
25
|
-
]
|
|
26
|
-
)
|
|
27
|
-
end
|
|
28
|
-
attr_accessor :guardrail_metrics
|
|
29
|
-
|
|
30
17
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
31
18
|
# contain at least a `user_prompt` field or a `system_prompt` field. For
|
|
32
19
|
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
@@ -44,13 +31,6 @@ module Deeprails
|
|
|
44
31
|
sig { returns(String) }
|
|
45
32
|
attr_accessor :model_output
|
|
46
33
|
|
|
47
|
-
# Model ID used to generate the output, like `gpt-4o` or `o3`.
|
|
48
|
-
sig { returns(T.nilable(String)) }
|
|
49
|
-
attr_reader :model_used
|
|
50
|
-
|
|
51
|
-
sig { params(model_used: String).void }
|
|
52
|
-
attr_writer :model_used
|
|
53
|
-
|
|
54
34
|
# An optional, user-defined tag for the event.
|
|
55
35
|
sig { returns(T.nilable(String)) }
|
|
56
36
|
attr_reader :nametag
|
|
@@ -78,32 +58,20 @@ module Deeprails
|
|
|
78
58
|
|
|
79
59
|
sig do
|
|
80
60
|
params(
|
|
81
|
-
guardrail_metrics:
|
|
82
|
-
T::Array[
|
|
83
|
-
Deeprails::MonitorSubmitEventParams::GuardrailMetric::OrSymbol
|
|
84
|
-
],
|
|
85
61
|
model_input: Deeprails::MonitorSubmitEventParams::ModelInput::OrHash,
|
|
86
62
|
model_output: String,
|
|
87
|
-
model_used: String,
|
|
88
63
|
nametag: String,
|
|
89
64
|
run_mode: Deeprails::MonitorSubmitEventParams::RunMode::OrSymbol,
|
|
90
65
|
request_options: Deeprails::RequestOptions::OrHash
|
|
91
66
|
).returns(T.attached_class)
|
|
92
67
|
end
|
|
93
68
|
def self.new(
|
|
94
|
-
# An array of guardrail metrics that the model input and output pair will be
|
|
95
|
-
# evaluated on. For non-enterprise users, these will be limited to `correctness`,
|
|
96
|
-
# `completeness`, `instruction_adherence`, `context_adherence`,
|
|
97
|
-
# `ground_truth_adherence`, and/or `comprehensive_safety`.
|
|
98
|
-
guardrail_metrics:,
|
|
99
69
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
100
70
|
# contain at least a `user_prompt` field or a `system_prompt` field. For
|
|
101
71
|
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
102
72
|
model_input:,
|
|
103
73
|
# Output generated by the LLM to be evaluated.
|
|
104
74
|
model_output:,
|
|
105
|
-
# Model ID used to generate the output, like `gpt-4o` or `o3`.
|
|
106
|
-
model_used: nil,
|
|
107
75
|
# An optional, user-defined tag for the event.
|
|
108
76
|
nametag: nil,
|
|
109
77
|
# Run mode for the monitor event. The run mode allows the user to optimize for
|
|
@@ -118,13 +86,8 @@ module Deeprails
|
|
|
118
86
|
sig do
|
|
119
87
|
override.returns(
|
|
120
88
|
{
|
|
121
|
-
guardrail_metrics:
|
|
122
|
-
T::Array[
|
|
123
|
-
Deeprails::MonitorSubmitEventParams::GuardrailMetric::OrSymbol
|
|
124
|
-
],
|
|
125
89
|
model_input: Deeprails::MonitorSubmitEventParams::ModelInput,
|
|
126
90
|
model_output: String,
|
|
127
|
-
model_used: String,
|
|
128
91
|
nametag: String,
|
|
129
92
|
run_mode: Deeprails::MonitorSubmitEventParams::RunMode::OrSymbol,
|
|
130
93
|
request_options: Deeprails::RequestOptions
|
|
@@ -134,57 +97,6 @@ module Deeprails
|
|
|
134
97
|
def to_hash
|
|
135
98
|
end
|
|
136
99
|
|
|
137
|
-
module GuardrailMetric
|
|
138
|
-
extend Deeprails::Internal::Type::Enum
|
|
139
|
-
|
|
140
|
-
TaggedSymbol =
|
|
141
|
-
T.type_alias do
|
|
142
|
-
T.all(Symbol, Deeprails::MonitorSubmitEventParams::GuardrailMetric)
|
|
143
|
-
end
|
|
144
|
-
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
145
|
-
|
|
146
|
-
CORRECTNESS =
|
|
147
|
-
T.let(
|
|
148
|
-
:correctness,
|
|
149
|
-
Deeprails::MonitorSubmitEventParams::GuardrailMetric::TaggedSymbol
|
|
150
|
-
)
|
|
151
|
-
COMPLETENESS =
|
|
152
|
-
T.let(
|
|
153
|
-
:completeness,
|
|
154
|
-
Deeprails::MonitorSubmitEventParams::GuardrailMetric::TaggedSymbol
|
|
155
|
-
)
|
|
156
|
-
INSTRUCTION_ADHERENCE =
|
|
157
|
-
T.let(
|
|
158
|
-
:instruction_adherence,
|
|
159
|
-
Deeprails::MonitorSubmitEventParams::GuardrailMetric::TaggedSymbol
|
|
160
|
-
)
|
|
161
|
-
CONTEXT_ADHERENCE =
|
|
162
|
-
T.let(
|
|
163
|
-
:context_adherence,
|
|
164
|
-
Deeprails::MonitorSubmitEventParams::GuardrailMetric::TaggedSymbol
|
|
165
|
-
)
|
|
166
|
-
GROUND_TRUTH_ADHERENCE =
|
|
167
|
-
T.let(
|
|
168
|
-
:ground_truth_adherence,
|
|
169
|
-
Deeprails::MonitorSubmitEventParams::GuardrailMetric::TaggedSymbol
|
|
170
|
-
)
|
|
171
|
-
COMPREHENSIVE_SAFETY =
|
|
172
|
-
T.let(
|
|
173
|
-
:comprehensive_safety,
|
|
174
|
-
Deeprails::MonitorSubmitEventParams::GuardrailMetric::TaggedSymbol
|
|
175
|
-
)
|
|
176
|
-
|
|
177
|
-
sig do
|
|
178
|
-
override.returns(
|
|
179
|
-
T::Array[
|
|
180
|
-
Deeprails::MonitorSubmitEventParams::GuardrailMetric::TaggedSymbol
|
|
181
|
-
]
|
|
182
|
-
)
|
|
183
|
-
end
|
|
184
|
-
def self.values
|
|
185
|
-
end
|
|
186
|
-
end
|
|
187
|
-
|
|
188
100
|
class ModelInput < Deeprails::Internal::Type::BaseModel
|
|
189
101
|
OrHash =
|
|
190
102
|
T.type_alias do
|
|
@@ -7,16 +7,30 @@ module Deeprails
|
|
|
7
7
|
# using guardrails
|
|
8
8
|
sig do
|
|
9
9
|
params(
|
|
10
|
+
guardrail_metrics:
|
|
11
|
+
T::Array[Deeprails::MonitorCreateParams::GuardrailMetric::OrSymbol],
|
|
10
12
|
name: String,
|
|
11
13
|
description: String,
|
|
14
|
+
file_search: T::Array[String],
|
|
15
|
+
web_search: T::Boolean,
|
|
12
16
|
request_options: Deeprails::RequestOptions::OrHash
|
|
13
17
|
).returns(Deeprails::MonitorResponse)
|
|
14
18
|
end
|
|
15
19
|
def create(
|
|
20
|
+
# An array of guardrail metrics that the model input and output pair will be
|
|
21
|
+
# evaluated on. For non-enterprise users, these will be limited to `correctness`,
|
|
22
|
+
# `completeness`, `instruction_adherence`, `context_adherence`,
|
|
23
|
+
# `ground_truth_adherence`, and/or `comprehensive_safety`.
|
|
24
|
+
guardrail_metrics:,
|
|
16
25
|
# Name of the new monitor.
|
|
17
26
|
name:,
|
|
18
27
|
# Description of the new monitor.
|
|
19
28
|
description: nil,
|
|
29
|
+
# An array of file IDs to search in the monitor's evaluations. Files must be
|
|
30
|
+
# uploaded via the DeepRails API first.
|
|
31
|
+
file_search: nil,
|
|
32
|
+
# Whether to enable web search for this monitor's evaluations. Defaults to false.
|
|
33
|
+
web_search: nil,
|
|
20
34
|
request_options: {}
|
|
21
35
|
)
|
|
22
36
|
end
|
|
@@ -70,13 +84,8 @@ module Deeprails
|
|
|
70
84
|
sig do
|
|
71
85
|
params(
|
|
72
86
|
monitor_id: String,
|
|
73
|
-
guardrail_metrics:
|
|
74
|
-
T::Array[
|
|
75
|
-
Deeprails::MonitorSubmitEventParams::GuardrailMetric::OrSymbol
|
|
76
|
-
],
|
|
77
87
|
model_input: Deeprails::MonitorSubmitEventParams::ModelInput::OrHash,
|
|
78
88
|
model_output: String,
|
|
79
|
-
model_used: String,
|
|
80
89
|
nametag: String,
|
|
81
90
|
run_mode: Deeprails::MonitorSubmitEventParams::RunMode::OrSymbol,
|
|
82
91
|
request_options: Deeprails::RequestOptions::OrHash
|
|
@@ -85,19 +94,12 @@ module Deeprails
|
|
|
85
94
|
def submit_event(
|
|
86
95
|
# The ID of the monitor associated with this event.
|
|
87
96
|
monitor_id,
|
|
88
|
-
# An array of guardrail metrics that the model input and output pair will be
|
|
89
|
-
# evaluated on. For non-enterprise users, these will be limited to `correctness`,
|
|
90
|
-
# `completeness`, `instruction_adherence`, `context_adherence`,
|
|
91
|
-
# `ground_truth_adherence`, and/or `comprehensive_safety`.
|
|
92
|
-
guardrail_metrics:,
|
|
93
97
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
94
98
|
# contain at least a `user_prompt` field or a `system_prompt` field. For
|
|
95
99
|
# ground_truth_adherence guardrail metric, `ground_truth` should be provided.
|
|
96
100
|
model_input:,
|
|
97
101
|
# Output generated by the LLM to be evaluated.
|
|
98
102
|
model_output:,
|
|
99
|
-
# Model ID used to generate the output, like `gpt-4o` or `o3`.
|
|
100
|
-
model_used: nil,
|
|
101
103
|
# An optional, user-defined tag for the event.
|
|
102
104
|
nametag: nil,
|
|
103
105
|
# Run mode for the monitor event. The run mode allows the user to optimize for
|
|
@@ -1,30 +1,73 @@
|
|
|
1
1
|
module Deeprails
|
|
2
2
|
module Models
|
|
3
3
|
type monitor_create_params =
|
|
4
|
-
{
|
|
4
|
+
{
|
|
5
|
+
guardrail_metrics: ::Array[Deeprails::Models::MonitorCreateParams::guardrail_metric],
|
|
6
|
+
name: String,
|
|
7
|
+
description: String,
|
|
8
|
+
file_search: ::Array[String],
|
|
9
|
+
web_search: bool
|
|
10
|
+
}
|
|
5
11
|
& Deeprails::Internal::Type::request_parameters
|
|
6
12
|
|
|
7
13
|
class MonitorCreateParams < Deeprails::Internal::Type::BaseModel
|
|
8
14
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
9
15
|
include Deeprails::Internal::Type::RequestParameters
|
|
10
16
|
|
|
17
|
+
attr_accessor guardrail_metrics: ::Array[Deeprails::Models::MonitorCreateParams::guardrail_metric]
|
|
18
|
+
|
|
11
19
|
attr_accessor name: String
|
|
12
20
|
|
|
13
21
|
attr_reader description: String?
|
|
14
22
|
|
|
15
23
|
def description=: (String) -> String
|
|
16
24
|
|
|
25
|
+
attr_reader file_search: ::Array[String]?
|
|
26
|
+
|
|
27
|
+
def file_search=: (::Array[String]) -> ::Array[String]
|
|
28
|
+
|
|
29
|
+
attr_reader web_search: bool?
|
|
30
|
+
|
|
31
|
+
def web_search=: (bool) -> bool
|
|
32
|
+
|
|
17
33
|
def initialize: (
|
|
34
|
+
guardrail_metrics: ::Array[Deeprails::Models::MonitorCreateParams::guardrail_metric],
|
|
18
35
|
name: String,
|
|
19
36
|
?description: String,
|
|
37
|
+
?file_search: ::Array[String],
|
|
38
|
+
?web_search: bool,
|
|
20
39
|
?request_options: Deeprails::request_opts
|
|
21
40
|
) -> void
|
|
22
41
|
|
|
23
42
|
def to_hash: -> {
|
|
43
|
+
guardrail_metrics: ::Array[Deeprails::Models::MonitorCreateParams::guardrail_metric],
|
|
24
44
|
name: String,
|
|
25
45
|
description: String,
|
|
46
|
+
file_search: ::Array[String],
|
|
47
|
+
web_search: bool,
|
|
26
48
|
request_options: Deeprails::RequestOptions
|
|
27
49
|
}
|
|
50
|
+
|
|
51
|
+
type guardrail_metric =
|
|
52
|
+
:correctness
|
|
53
|
+
| :completeness
|
|
54
|
+
| :instruction_adherence
|
|
55
|
+
| :context_adherence
|
|
56
|
+
| :ground_truth_adherence
|
|
57
|
+
| :comprehensive_safety
|
|
58
|
+
|
|
59
|
+
module GuardrailMetric
|
|
60
|
+
extend Deeprails::Internal::Type::Enum
|
|
61
|
+
|
|
62
|
+
CORRECTNESS: :correctness
|
|
63
|
+
COMPLETENESS: :completeness
|
|
64
|
+
INSTRUCTION_ADHERENCE: :instruction_adherence
|
|
65
|
+
CONTEXT_ADHERENCE: :context_adherence
|
|
66
|
+
GROUND_TRUTH_ADHERENCE: :ground_truth_adherence
|
|
67
|
+
COMPREHENSIVE_SAFETY: :comprehensive_safety
|
|
68
|
+
|
|
69
|
+
def self?.values: -> ::Array[Deeprails::Models::MonitorCreateParams::guardrail_metric]
|
|
70
|
+
end
|
|
28
71
|
end
|
|
29
72
|
end
|
|
30
73
|
end
|
|
@@ -2,10 +2,8 @@ module Deeprails
|
|
|
2
2
|
module Models
|
|
3
3
|
type monitor_submit_event_params =
|
|
4
4
|
{
|
|
5
|
-
guardrail_metrics: ::Array[Deeprails::Models::MonitorSubmitEventParams::guardrail_metric],
|
|
6
5
|
model_input: Deeprails::MonitorSubmitEventParams::ModelInput,
|
|
7
6
|
model_output: String,
|
|
8
|
-
model_used: String,
|
|
9
7
|
nametag: String,
|
|
10
8
|
run_mode: Deeprails::Models::MonitorSubmitEventParams::run_mode
|
|
11
9
|
}
|
|
@@ -15,16 +13,10 @@ module Deeprails
|
|
|
15
13
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
16
14
|
include Deeprails::Internal::Type::RequestParameters
|
|
17
15
|
|
|
18
|
-
attr_accessor guardrail_metrics: ::Array[Deeprails::Models::MonitorSubmitEventParams::guardrail_metric]
|
|
19
|
-
|
|
20
16
|
attr_accessor model_input: Deeprails::MonitorSubmitEventParams::ModelInput
|
|
21
17
|
|
|
22
18
|
attr_accessor model_output: String
|
|
23
19
|
|
|
24
|
-
attr_reader model_used: String?
|
|
25
|
-
|
|
26
|
-
def model_used=: (String) -> String
|
|
27
|
-
|
|
28
20
|
attr_reader nametag: String?
|
|
29
21
|
|
|
30
22
|
def nametag=: (String) -> String
|
|
@@ -36,46 +28,21 @@ module Deeprails
|
|
|
36
28
|
) -> Deeprails::Models::MonitorSubmitEventParams::run_mode
|
|
37
29
|
|
|
38
30
|
def initialize: (
|
|
39
|
-
guardrail_metrics: ::Array[Deeprails::Models::MonitorSubmitEventParams::guardrail_metric],
|
|
40
31
|
model_input: Deeprails::MonitorSubmitEventParams::ModelInput,
|
|
41
32
|
model_output: String,
|
|
42
|
-
?model_used: String,
|
|
43
33
|
?nametag: String,
|
|
44
34
|
?run_mode: Deeprails::Models::MonitorSubmitEventParams::run_mode,
|
|
45
35
|
?request_options: Deeprails::request_opts
|
|
46
36
|
) -> void
|
|
47
37
|
|
|
48
38
|
def to_hash: -> {
|
|
49
|
-
guardrail_metrics: ::Array[Deeprails::Models::MonitorSubmitEventParams::guardrail_metric],
|
|
50
39
|
model_input: Deeprails::MonitorSubmitEventParams::ModelInput,
|
|
51
40
|
model_output: String,
|
|
52
|
-
model_used: String,
|
|
53
41
|
nametag: String,
|
|
54
42
|
run_mode: Deeprails::Models::MonitorSubmitEventParams::run_mode,
|
|
55
43
|
request_options: Deeprails::RequestOptions
|
|
56
44
|
}
|
|
57
45
|
|
|
58
|
-
type guardrail_metric =
|
|
59
|
-
:correctness
|
|
60
|
-
| :completeness
|
|
61
|
-
| :instruction_adherence
|
|
62
|
-
| :context_adherence
|
|
63
|
-
| :ground_truth_adherence
|
|
64
|
-
| :comprehensive_safety
|
|
65
|
-
|
|
66
|
-
module GuardrailMetric
|
|
67
|
-
extend Deeprails::Internal::Type::Enum
|
|
68
|
-
|
|
69
|
-
CORRECTNESS: :correctness
|
|
70
|
-
COMPLETENESS: :completeness
|
|
71
|
-
INSTRUCTION_ADHERENCE: :instruction_adherence
|
|
72
|
-
CONTEXT_ADHERENCE: :context_adherence
|
|
73
|
-
GROUND_TRUTH_ADHERENCE: :ground_truth_adherence
|
|
74
|
-
COMPREHENSIVE_SAFETY: :comprehensive_safety
|
|
75
|
-
|
|
76
|
-
def self?.values: -> ::Array[Deeprails::Models::MonitorSubmitEventParams::guardrail_metric]
|
|
77
|
-
end
|
|
78
|
-
|
|
79
46
|
type model_input =
|
|
80
47
|
{ ground_truth: String, system_prompt: String, user_prompt: String }
|
|
81
48
|
|
|
@@ -2,8 +2,11 @@ module Deeprails
|
|
|
2
2
|
module Resources
|
|
3
3
|
class Monitor
|
|
4
4
|
def create: (
|
|
5
|
+
guardrail_metrics: ::Array[Deeprails::Models::MonitorCreateParams::guardrail_metric],
|
|
5
6
|
name: String,
|
|
6
7
|
?description: String,
|
|
8
|
+
?file_search: ::Array[String],
|
|
9
|
+
?web_search: bool,
|
|
7
10
|
?request_options: Deeprails::request_opts
|
|
8
11
|
) -> Deeprails::MonitorResponse
|
|
9
12
|
|
|
@@ -23,10 +26,8 @@ module Deeprails
|
|
|
23
26
|
|
|
24
27
|
def submit_event: (
|
|
25
28
|
String monitor_id,
|
|
26
|
-
guardrail_metrics: ::Array[Deeprails::Models::MonitorSubmitEventParams::guardrail_metric],
|
|
27
29
|
model_input: Deeprails::MonitorSubmitEventParams::ModelInput,
|
|
28
30
|
model_output: String,
|
|
29
|
-
?model_used: String,
|
|
30
31
|
?nametag: String,
|
|
31
32
|
?run_mode: Deeprails::Models::MonitorSubmitEventParams::run_mode,
|
|
32
33
|
?request_options: Deeprails::request_opts
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: deeprails
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.11.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Deeprails
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2025-
|
|
11
|
+
date: 2025-11-02 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: connection_pool
|