deeprails 0.18.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +92 -0
- data/README.md +1 -1
- data/lib/deeprails/internal/stream.rb +29 -0
- data/lib/deeprails/internal/transport/base_client.rb +3 -1
- data/lib/deeprails/internal/transport/pooled_net_requester.rb +13 -11
- data/lib/deeprails/internal/type/base_stream.rb +83 -0
- data/lib/deeprails/internal/util.rb +39 -3
- data/lib/deeprails/models/defend_create_workflow_params.rb +14 -1
- data/lib/deeprails/models/defend_response.rb +134 -10
- data/lib/deeprails/models/defend_retrieve_event_params.rb +7 -1
- data/lib/deeprails/models/defend_retrieve_workflow_params.rb +8 -1
- data/lib/deeprails/models/defend_submit_and_stream_event_params.rb +92 -0
- data/lib/deeprails/models/defend_submit_and_stream_event_response.rb +7 -0
- data/lib/deeprails/models/defend_submit_event_params.rb +70 -20
- data/lib/deeprails/models/defend_update_response.rb +9 -1
- data/lib/deeprails/models/defend_update_workflow_params.rb +138 -5
- data/lib/deeprails/models/monitor_create_params.rb +14 -1
- data/lib/deeprails/models/monitor_detail_response.rb +60 -17
- data/lib/deeprails/models/monitor_event_detail_response.rb +11 -5
- data/lib/deeprails/models/monitor_retrieve_event_params.rb +7 -1
- data/lib/deeprails/models/monitor_retrieve_params.rb +8 -1
- data/lib/deeprails/models/monitor_submit_event_params.rb +70 -20
- data/lib/deeprails/models/monitor_update_params.rb +53 -5
- data/lib/deeprails/models/workflow_event_detail_response.rb +99 -16
- data/lib/deeprails/models/workflow_event_response.rb +9 -1
- data/lib/deeprails/models.rb +2 -0
- data/lib/deeprails/resources/defend.rb +70 -5
- data/lib/deeprails/resources/monitor.rb +16 -7
- data/lib/deeprails/version.rb +1 -1
- data/lib/deeprails.rb +5 -0
- data/manifest.yaml +1 -0
- data/rbi/deeprails/internal/stream.rbi +20 -0
- data/rbi/deeprails/internal/transport/base_client.rbi +18 -2
- data/rbi/deeprails/internal/type/base_stream.rbi +75 -0
- data/rbi/deeprails/internal/util.rbi +21 -1
- data/rbi/deeprails/models/defend_create_workflow_params.rbi +21 -0
- data/rbi/deeprails/models/defend_response.rbi +286 -14
- data/rbi/deeprails/models/defend_retrieve_event_params.rbi +10 -2
- data/rbi/deeprails/models/defend_retrieve_workflow_params.rbi +10 -1
- data/rbi/deeprails/models/defend_submit_and_stream_event_params.rbi +155 -0
- data/rbi/deeprails/models/defend_submit_and_stream_event_response.rbi +7 -0
- data/rbi/deeprails/models/defend_submit_event_params.rbi +134 -31
- data/rbi/deeprails/models/defend_update_response.rbi +14 -3
- data/rbi/deeprails/models/defend_update_workflow_params.rbi +304 -4
- data/rbi/deeprails/models/monitor_create_params.rbi +21 -0
- data/rbi/deeprails/models/monitor_detail_response.rbi +122 -28
- data/rbi/deeprails/models/monitor_event_detail_response.rbi +20 -8
- data/rbi/deeprails/models/monitor_retrieve_event_params.rbi +10 -2
- data/rbi/deeprails/models/monitor_retrieve_params.rbi +10 -1
- data/rbi/deeprails/models/monitor_submit_event_params.rbi +134 -31
- data/rbi/deeprails/models/monitor_update_params.rbi +113 -4
- data/rbi/deeprails/models/workflow_event_detail_response.rbi +204 -24
- data/rbi/deeprails/models/workflow_event_response.rbi +8 -0
- data/rbi/deeprails/models.rbi +3 -0
- data/rbi/deeprails/resources/defend.rbi +101 -6
- data/rbi/deeprails/resources/monitor.rbi +31 -8
- data/sig/deeprails/internal/stream.rbs +9 -0
- data/sig/deeprails/internal/type/base_stream.rbs +38 -0
- data/sig/deeprails/internal/util.rbs +10 -0
- data/sig/deeprails/models/defend_create_workflow_params.rbs +7 -0
- data/sig/deeprails/models/defend_response.rbs +114 -8
- data/sig/deeprails/models/defend_retrieve_event_params.rbs +6 -1
- data/sig/deeprails/models/defend_retrieve_workflow_params.rbs +6 -1
- data/sig/deeprails/models/defend_submit_and_stream_event_params.rbs +73 -0
- data/sig/deeprails/models/defend_submit_and_stream_event_response.rbs +5 -0
- data/sig/deeprails/models/defend_submit_event_params.rbs +53 -13
- data/sig/deeprails/models/defend_update_response.rbs +10 -3
- data/sig/deeprails/models/defend_update_workflow_params.rbs +108 -1
- data/sig/deeprails/models/monitor_create_params.rbs +7 -0
- data/sig/deeprails/models/monitor_detail_response.rbs +48 -13
- data/sig/deeprails/models/monitor_event_detail_response.rbs +12 -4
- data/sig/deeprails/models/monitor_retrieve_event_params.rbs +6 -1
- data/sig/deeprails/models/monitor_retrieve_params.rbs +6 -1
- data/sig/deeprails/models/monitor_submit_event_params.rbs +53 -13
- data/sig/deeprails/models/monitor_update_params.rbs +50 -1
- data/sig/deeprails/models/workflow_event_detail_response.rbs +100 -15
- data/sig/deeprails/models/workflow_event_response.rbs +5 -0
- data/sig/deeprails/models.rbs +2 -0
- data/sig/deeprails/resources/defend.rbs +20 -0
- data/sig/deeprails/resources/monitor.rbs +4 -0
- metadata +28 -2
|
@@ -9,12 +9,14 @@ module Deeprails
|
|
|
9
9
|
# Use this endpoint to create a new monitor to evaluate model inputs and outputs
|
|
10
10
|
# using guardrails
|
|
11
11
|
#
|
|
12
|
-
# @overload create(guardrail_metrics:, name:, description: nil, file_search: nil, web_search: nil, request_options: {})
|
|
12
|
+
# @overload create(guardrail_metrics:, name:, context_awareness: nil, description: nil, file_search: nil, web_search: nil, request_options: {})
|
|
13
13
|
#
|
|
14
14
|
# @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorCreateParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
|
|
15
15
|
#
|
|
16
16
|
# @param name [String] Name of the new monitor.
|
|
17
17
|
#
|
|
18
|
+
# @param context_awareness [Boolean] Context includes any structured information that directly relates to the model’s
|
|
19
|
+
#
|
|
18
20
|
# @param description [String] Description of the new monitor.
|
|
19
21
|
#
|
|
20
22
|
# @param file_search [Array<String>] An array of file IDs to search in the monitor's evaluations. Files must be uploa
|
|
@@ -56,10 +58,11 @@ module Deeprails
|
|
|
56
58
|
# @see Deeprails::Models::MonitorRetrieveParams
|
|
57
59
|
def retrieve(monitor_id, params = {})
|
|
58
60
|
parsed, options = Deeprails::MonitorRetrieveParams.dump_request(params)
|
|
61
|
+
query = Deeprails::Internal::Util.encode_query_params(parsed)
|
|
59
62
|
@client.request(
|
|
60
63
|
method: :get,
|
|
61
64
|
path: ["monitor/%1$s", monitor_id],
|
|
62
|
-
query:
|
|
65
|
+
query: query,
|
|
63
66
|
model: Deeprails::MonitorDetailResponse,
|
|
64
67
|
options: options
|
|
65
68
|
)
|
|
@@ -68,19 +71,25 @@ module Deeprails
|
|
|
68
71
|
# Some parameter documentations has been truncated, see
|
|
69
72
|
# {Deeprails::Models::MonitorUpdateParams} for more details.
|
|
70
73
|
#
|
|
71
|
-
# Use this endpoint to update the name,
|
|
72
|
-
# monitor
|
|
74
|
+
# Use this endpoint to update the name, status, and/or other details of an
|
|
75
|
+
# existing monitor.
|
|
73
76
|
#
|
|
74
|
-
# @overload update(monitor_id, description: nil, name: nil, status: nil, request_options: {})
|
|
77
|
+
# @overload update(monitor_id, description: nil, file_search: nil, guardrail_metrics: nil, name: nil, status: nil, web_search: nil, request_options: {})
|
|
75
78
|
#
|
|
76
79
|
# @param monitor_id [String] The ID of the monitor to edit.
|
|
77
80
|
#
|
|
78
|
-
# @param description [String]
|
|
81
|
+
# @param description [String] New description of the monitor.
|
|
82
|
+
#
|
|
83
|
+
# @param file_search [Array<String>] An array of file IDs to search in the monitor's evaluations. Files must be uploa
|
|
79
84
|
#
|
|
80
|
-
# @param
|
|
85
|
+
# @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorUpdateParams::GuardrailMetric>] An array of the new guardrail metrics that model input and output pairs will be
|
|
86
|
+
#
|
|
87
|
+
# @param name [String] New name of the monitor.
|
|
81
88
|
#
|
|
82
89
|
# @param status [Symbol, Deeprails::Models::MonitorUpdateParams::Status] Status of the monitor. Can be `active` or `inactive`. Inactive monitors no lon
|
|
83
90
|
#
|
|
91
|
+
# @param web_search [Boolean] Whether to enable web search for this monitor's evaluations.
|
|
92
|
+
#
|
|
84
93
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
85
94
|
#
|
|
86
95
|
# @return [Deeprails::Models::MonitorUpdateResponse]
|
data/lib/deeprails/version.rb
CHANGED
data/lib/deeprails.rb
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
# Standard libraries.
|
|
4
4
|
# rubocop:disable Lint/RedundantRequireStatement
|
|
5
5
|
require "English"
|
|
6
|
+
require "base64"
|
|
6
7
|
require "cgi"
|
|
7
8
|
require "date"
|
|
8
9
|
require "erb"
|
|
@@ -43,6 +44,7 @@ require_relative "deeprails/internal/type/array_of"
|
|
|
43
44
|
require_relative "deeprails/internal/type/hash_of"
|
|
44
45
|
require_relative "deeprails/internal/type/base_model"
|
|
45
46
|
require_relative "deeprails/internal/type/base_page"
|
|
47
|
+
require_relative "deeprails/internal/type/base_stream"
|
|
46
48
|
require_relative "deeprails/internal/type/request_parameters"
|
|
47
49
|
require_relative "deeprails/internal"
|
|
48
50
|
require_relative "deeprails/request_options"
|
|
@@ -51,11 +53,14 @@ require_relative "deeprails/errors"
|
|
|
51
53
|
require_relative "deeprails/internal/transport/base_client"
|
|
52
54
|
require_relative "deeprails/internal/transport/pooled_net_requester"
|
|
53
55
|
require_relative "deeprails/client"
|
|
56
|
+
require_relative "deeprails/internal/stream"
|
|
54
57
|
require_relative "deeprails/models/defend_create_response"
|
|
55
58
|
require_relative "deeprails/models/defend_create_workflow_params"
|
|
56
59
|
require_relative "deeprails/models/defend_response"
|
|
57
60
|
require_relative "deeprails/models/defend_retrieve_event_params"
|
|
58
61
|
require_relative "deeprails/models/defend_retrieve_workflow_params"
|
|
62
|
+
require_relative "deeprails/models/defend_submit_and_stream_event_params"
|
|
63
|
+
require_relative "deeprails/models/defend_submit_and_stream_event_response"
|
|
59
64
|
require_relative "deeprails/models/defend_submit_event_params"
|
|
60
65
|
require_relative "deeprails/models/defend_update_response"
|
|
61
66
|
require_relative "deeprails/models/defend_update_workflow_params"
|
data/manifest.yaml
CHANGED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# typed: strong
|
|
2
|
+
|
|
3
|
+
module Deeprails
|
|
4
|
+
module Internal
|
|
5
|
+
class Stream
|
|
6
|
+
Message =
|
|
7
|
+
type_member(:in) do
|
|
8
|
+
{ fixed: Deeprails::Internal::Util::ServerSentEvent }
|
|
9
|
+
end
|
|
10
|
+
Elem = type_member(:out)
|
|
11
|
+
|
|
12
|
+
include Deeprails::Internal::Type::BaseStream
|
|
13
|
+
|
|
14
|
+
# @api private
|
|
15
|
+
sig { override.returns(T::Enumerable[Elem]) }
|
|
16
|
+
private def iterator
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -49,7 +49,15 @@ module Deeprails
|
|
|
49
49
|
]
|
|
50
50
|
]
|
|
51
51
|
),
|
|
52
|
-
stream:
|
|
52
|
+
stream:
|
|
53
|
+
T.nilable(
|
|
54
|
+
T::Class[
|
|
55
|
+
Deeprails::Internal::Type::BaseStream[
|
|
56
|
+
T.anything,
|
|
57
|
+
Deeprails::Internal::Type::BaseModel
|
|
58
|
+
]
|
|
59
|
+
]
|
|
60
|
+
),
|
|
53
61
|
model: T.nilable(Deeprails::Internal::Type::Converter::Input),
|
|
54
62
|
options: T.nilable(Deeprails::RequestOptions::OrHash)
|
|
55
63
|
}
|
|
@@ -269,7 +277,15 @@ module Deeprails
|
|
|
269
277
|
]
|
|
270
278
|
]
|
|
271
279
|
),
|
|
272
|
-
stream:
|
|
280
|
+
stream:
|
|
281
|
+
T.nilable(
|
|
282
|
+
T::Class[
|
|
283
|
+
Deeprails::Internal::Type::BaseStream[
|
|
284
|
+
T.anything,
|
|
285
|
+
Deeprails::Internal::Type::BaseModel
|
|
286
|
+
]
|
|
287
|
+
]
|
|
288
|
+
),
|
|
273
289
|
model: T.nilable(Deeprails::Internal::Type::Converter::Input),
|
|
274
290
|
options: T.nilable(Deeprails::RequestOptions::OrHash)
|
|
275
291
|
).returns(T.anything)
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# typed: strong
|
|
2
|
+
|
|
3
|
+
module Deeprails
|
|
4
|
+
module Internal
|
|
5
|
+
module Type
|
|
6
|
+
# @api private
|
|
7
|
+
#
|
|
8
|
+
# This module provides a base implementation for streaming responses in the SDK.
|
|
9
|
+
module BaseStream
|
|
10
|
+
include Enumerable
|
|
11
|
+
|
|
12
|
+
Message = type_member(:in)
|
|
13
|
+
Elem = type_member(:out)
|
|
14
|
+
|
|
15
|
+
sig { returns(Integer) }
|
|
16
|
+
attr_reader :status
|
|
17
|
+
|
|
18
|
+
sig { returns(T::Hash[String, String]) }
|
|
19
|
+
attr_reader :headers
|
|
20
|
+
|
|
21
|
+
sig { void }
|
|
22
|
+
def close
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# @api private
|
|
26
|
+
sig { overridable.returns(T::Enumerable[Elem]) }
|
|
27
|
+
private def iterator
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
sig { params(blk: T.proc.params(arg0: Elem).void).void }
|
|
31
|
+
def each(&blk)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
sig { returns(T::Enumerator[Elem]) }
|
|
35
|
+
def to_enum
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# @api private
|
|
39
|
+
sig do
|
|
40
|
+
params(
|
|
41
|
+
model:
|
|
42
|
+
T.any(T::Class[T.anything], Deeprails::Internal::Type::Converter),
|
|
43
|
+
url: URI::Generic,
|
|
44
|
+
status: Integer,
|
|
45
|
+
headers: T::Hash[String, String],
|
|
46
|
+
response: Net::HTTPResponse,
|
|
47
|
+
unwrap:
|
|
48
|
+
T.any(
|
|
49
|
+
Symbol,
|
|
50
|
+
Integer,
|
|
51
|
+
T::Array[T.any(Symbol, Integer)],
|
|
52
|
+
T.proc.params(arg0: T.anything).returns(T.anything)
|
|
53
|
+
),
|
|
54
|
+
stream: T::Enumerable[Message]
|
|
55
|
+
).void
|
|
56
|
+
end
|
|
57
|
+
def initialize(
|
|
58
|
+
model:,
|
|
59
|
+
url:,
|
|
60
|
+
status:,
|
|
61
|
+
headers:,
|
|
62
|
+
response:,
|
|
63
|
+
unwrap:,
|
|
64
|
+
stream:
|
|
65
|
+
)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# @api private
|
|
69
|
+
sig { returns(String) }
|
|
70
|
+
def inspect
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
@@ -296,11 +296,31 @@ module Deeprails
|
|
|
296
296
|
end
|
|
297
297
|
|
|
298
298
|
JSON_CONTENT =
|
|
299
|
-
T.let(%r{^application/(?:
|
|
299
|
+
T.let(%r{^application/(?:[a-zA-Z0-9.-]+\+)?json(?!l)}, Regexp)
|
|
300
300
|
JSONL_CONTENT =
|
|
301
301
|
T.let(%r{^application/(:?x-(?:n|l)djson)|(:?(?:x-)?jsonl)}, Regexp)
|
|
302
302
|
|
|
303
303
|
class << self
|
|
304
|
+
# @api private
|
|
305
|
+
sig do
|
|
306
|
+
params(query: Deeprails::Internal::AnyHash).returns(
|
|
307
|
+
Deeprails::Internal::AnyHash
|
|
308
|
+
)
|
|
309
|
+
end
|
|
310
|
+
def encode_query_params(query)
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
# @api private
|
|
314
|
+
sig do
|
|
315
|
+
params(
|
|
316
|
+
collection: Deeprails::Internal::AnyHash,
|
|
317
|
+
key: String,
|
|
318
|
+
element: T.anything
|
|
319
|
+
).void
|
|
320
|
+
end
|
|
321
|
+
private def write_query_param_element!(collection, key, element)
|
|
322
|
+
end
|
|
323
|
+
|
|
304
324
|
# @api private
|
|
305
325
|
sig do
|
|
306
326
|
params(
|
|
@@ -67,6 +67,18 @@ module Deeprails
|
|
|
67
67
|
end
|
|
68
68
|
attr_writer :automatic_hallucination_tolerance_levels
|
|
69
69
|
|
|
70
|
+
# Context includes any structured information that directly relates to the model’s
|
|
71
|
+
# input and expected output—e.g., the recent turn-by-turn history between an AI
|
|
72
|
+
# tutor and a student, facts or state passed through an agentic workflow, or other
|
|
73
|
+
# domain-specific signals your system already knows and wants the model to
|
|
74
|
+
# condition on. This field determines whether to enable context awareness for this
|
|
75
|
+
# workflow's evaluations. Defaults to false.
|
|
76
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
77
|
+
attr_reader :context_awareness
|
|
78
|
+
|
|
79
|
+
sig { params(context_awareness: T::Boolean).void }
|
|
80
|
+
attr_writer :context_awareness
|
|
81
|
+
|
|
70
82
|
# Mapping of guardrail metrics to floating point threshold values. Possible
|
|
71
83
|
# metrics are `correctness`, `completeness`, `instruction_adherence`,
|
|
72
84
|
# `context_adherence`, `ground_truth_adherence`, or `comprehensive_safety`.
|
|
@@ -122,6 +134,7 @@ module Deeprails
|
|
|
122
134
|
Symbol,
|
|
123
135
|
Deeprails::DefendCreateWorkflowParams::AutomaticHallucinationToleranceLevel::OrSymbol
|
|
124
136
|
],
|
|
137
|
+
context_awareness: T::Boolean,
|
|
125
138
|
custom_hallucination_threshold_values: T::Hash[Symbol, Float],
|
|
126
139
|
description: String,
|
|
127
140
|
file_search: T::Array[String],
|
|
@@ -150,6 +163,13 @@ module Deeprails
|
|
|
150
163
|
# `instruction_adherence`, `context_adherence`, `ground_truth_adherence`, or
|
|
151
164
|
# `comprehensive_safety`.
|
|
152
165
|
automatic_hallucination_tolerance_levels: nil,
|
|
166
|
+
# Context includes any structured information that directly relates to the model’s
|
|
167
|
+
# input and expected output—e.g., the recent turn-by-turn history between an AI
|
|
168
|
+
# tutor and a student, facts or state passed through an agentic workflow, or other
|
|
169
|
+
# domain-specific signals your system already knows and wants the model to
|
|
170
|
+
# condition on. This field determines whether to enable context awareness for this
|
|
171
|
+
# workflow's evaluations. Defaults to false.
|
|
172
|
+
context_awareness: nil,
|
|
153
173
|
# Mapping of guardrail metrics to floating point threshold values. Possible
|
|
154
174
|
# metrics are `correctness`, `completeness`, `instruction_adherence`,
|
|
155
175
|
# `context_adherence`, `ground_truth_adherence`, or `comprehensive_safety`.
|
|
@@ -181,6 +201,7 @@ module Deeprails
|
|
|
181
201
|
Symbol,
|
|
182
202
|
Deeprails::DefendCreateWorkflowParams::AutomaticHallucinationToleranceLevel::OrSymbol
|
|
183
203
|
],
|
|
204
|
+
context_awareness: T::Boolean,
|
|
184
205
|
custom_hallucination_threshold_values: T::Hash[Symbol, Float],
|
|
185
206
|
description: String,
|
|
186
207
|
file_search: T::Array[String],
|