deeprails 0.17.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +71 -0
- data/README.md +3 -3
- data/lib/deeprails/internal/stream.rb +29 -0
- data/lib/deeprails/internal/transport/base_client.rb +3 -1
- data/lib/deeprails/internal/transport/pooled_net_requester.rb +12 -10
- data/lib/deeprails/internal/type/base_stream.rb +83 -0
- data/lib/deeprails/internal/util.rb +7 -2
- data/lib/deeprails/models/defend_create_workflow_params.rb +14 -1
- data/lib/deeprails/models/defend_response.rb +136 -12
- data/lib/deeprails/models/defend_submit_and_stream_event_params.rb +83 -0
- data/lib/deeprails/models/defend_submit_and_stream_event_response.rb +7 -0
- data/lib/deeprails/models/defend_submit_event_params.rb +52 -16
- data/lib/deeprails/models/defend_update_response.rb +9 -1
- data/lib/deeprails/models/defend_update_workflow_params.rb +131 -5
- data/lib/deeprails/models/file_upload_params.rb +5 -5
- data/lib/deeprails/models/monitor_create_params.rb +14 -1
- data/lib/deeprails/models/monitor_detail_response.rb +56 -17
- data/lib/deeprails/models/monitor_event_detail_response.rb +6 -2
- data/lib/deeprails/models/monitor_submit_event_params.rb +52 -16
- data/lib/deeprails/models/monitor_update_params.rb +46 -5
- data/lib/deeprails/models/workflow_event_detail_response.rb +106 -34
- data/lib/deeprails/models/workflow_event_response.rb +9 -1
- data/lib/deeprails/models.rb +2 -0
- data/lib/deeprails/resources/defend.rb +70 -7
- data/lib/deeprails/resources/files.rb +2 -2
- data/lib/deeprails/resources/monitor.rb +14 -6
- data/lib/deeprails/version.rb +1 -1
- data/lib/deeprails.rb +5 -0
- data/manifest.yaml +1 -0
- data/rbi/deeprails/internal/stream.rbi +20 -0
- data/rbi/deeprails/internal/transport/base_client.rbi +18 -2
- data/rbi/deeprails/internal/type/base_stream.rbi +75 -0
- data/rbi/deeprails/models/defend_create_workflow_params.rbi +21 -0
- data/rbi/deeprails/models/defend_response.rbi +288 -16
- data/rbi/deeprails/models/defend_submit_and_stream_event_params.rbi +148 -0
- data/rbi/deeprails/models/defend_submit_and_stream_event_response.rbi +7 -0
- data/rbi/deeprails/models/defend_submit_event_params.rbi +108 -25
- data/rbi/deeprails/models/defend_update_response.rbi +14 -3
- data/rbi/deeprails/models/defend_update_workflow_params.rbi +298 -4
- data/rbi/deeprails/models/file_upload_params.rbi +9 -6
- data/rbi/deeprails/models/monitor_create_params.rbi +21 -0
- data/rbi/deeprails/models/monitor_detail_response.rbi +107 -26
- data/rbi/deeprails/models/monitor_event_detail_response.rbi +4 -2
- data/rbi/deeprails/models/monitor_submit_event_params.rbi +108 -25
- data/rbi/deeprails/models/monitor_update_params.rbi +107 -4
- data/rbi/deeprails/models/workflow_event_detail_response.rbi +209 -66
- data/rbi/deeprails/models/workflow_event_response.rbi +8 -0
- data/rbi/deeprails/models.rbi +3 -0
- data/rbi/deeprails/resources/defend.rbi +99 -9
- data/rbi/deeprails/resources/files.rbi +3 -3
- data/rbi/deeprails/resources/monitor.rbi +28 -8
- data/sig/deeprails/internal/stream.rbs +9 -0
- data/sig/deeprails/internal/type/base_stream.rbs +38 -0
- data/sig/deeprails/models/defend_create_workflow_params.rbs +7 -0
- data/sig/deeprails/models/defend_response.rbs +114 -8
- data/sig/deeprails/models/defend_submit_and_stream_event_params.rbs +74 -0
- data/sig/deeprails/models/defend_submit_and_stream_event_response.rbs +5 -0
- data/sig/deeprails/models/defend_submit_event_params.rbs +39 -10
- data/sig/deeprails/models/defend_update_response.rbs +10 -3
- data/sig/deeprails/models/defend_update_workflow_params.rbs +103 -1
- data/sig/deeprails/models/file_upload_params.rbs +4 -4
- data/sig/deeprails/models/monitor_create_params.rbs +7 -0
- data/sig/deeprails/models/monitor_detail_response.rbs +36 -9
- data/sig/deeprails/models/monitor_submit_event_params.rbs +39 -10
- data/sig/deeprails/models/monitor_update_params.rbs +45 -1
- data/sig/deeprails/models/workflow_event_detail_response.rbs +100 -27
- data/sig/deeprails/models/workflow_event_response.rbs +5 -0
- data/sig/deeprails/models.rbs +2 -0
- data/sig/deeprails/resources/defend.rbs +20 -0
- data/sig/deeprails/resources/files.rbs +1 -1
- data/sig/deeprails/resources/monitor.rbs +4 -0
- metadata +28 -2
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
# typed: strong
|
|
2
|
+
|
|
3
|
+
module Deeprails
|
|
4
|
+
module Internal
|
|
5
|
+
module Type
|
|
6
|
+
# @api private
|
|
7
|
+
#
|
|
8
|
+
# This module provides a base implementation for streaming responses in the SDK.
|
|
9
|
+
module BaseStream
|
|
10
|
+
include Enumerable
|
|
11
|
+
|
|
12
|
+
Message = type_member(:in)
|
|
13
|
+
Elem = type_member(:out)
|
|
14
|
+
|
|
15
|
+
sig { returns(Integer) }
|
|
16
|
+
attr_reader :status
|
|
17
|
+
|
|
18
|
+
sig { returns(T::Hash[String, String]) }
|
|
19
|
+
attr_reader :headers
|
|
20
|
+
|
|
21
|
+
sig { void }
|
|
22
|
+
def close
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# @api private
|
|
26
|
+
sig { overridable.returns(T::Enumerable[Elem]) }
|
|
27
|
+
private def iterator
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
sig { params(blk: T.proc.params(arg0: Elem).void).void }
|
|
31
|
+
def each(&blk)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
sig { returns(T::Enumerator[Elem]) }
|
|
35
|
+
def to_enum
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# @api private
|
|
39
|
+
sig do
|
|
40
|
+
params(
|
|
41
|
+
model:
|
|
42
|
+
T.any(T::Class[T.anything], Deeprails::Internal::Type::Converter),
|
|
43
|
+
url: URI::Generic,
|
|
44
|
+
status: Integer,
|
|
45
|
+
headers: T::Hash[String, String],
|
|
46
|
+
response: Net::HTTPResponse,
|
|
47
|
+
unwrap:
|
|
48
|
+
T.any(
|
|
49
|
+
Symbol,
|
|
50
|
+
Integer,
|
|
51
|
+
T::Array[T.any(Symbol, Integer)],
|
|
52
|
+
T.proc.params(arg0: T.anything).returns(T.anything)
|
|
53
|
+
),
|
|
54
|
+
stream: T::Enumerable[Message]
|
|
55
|
+
).void
|
|
56
|
+
end
|
|
57
|
+
def initialize(
|
|
58
|
+
model:,
|
|
59
|
+
url:,
|
|
60
|
+
status:,
|
|
61
|
+
headers:,
|
|
62
|
+
response:,
|
|
63
|
+
unwrap:,
|
|
64
|
+
stream:
|
|
65
|
+
)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# @api private
|
|
69
|
+
sig { returns(String) }
|
|
70
|
+
def inspect
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
@@ -67,6 +67,18 @@ module Deeprails
|
|
|
67
67
|
end
|
|
68
68
|
attr_writer :automatic_hallucination_tolerance_levels
|
|
69
69
|
|
|
70
|
+
# Context includes any structured information that directly relates to the model’s
|
|
71
|
+
# input and expected output—e.g., the recent turn-by-turn history between an AI
|
|
72
|
+
# tutor and a student, facts or state passed through an agentic workflow, or other
|
|
73
|
+
# domain-specific signals your system already knows and wants the model to
|
|
74
|
+
# condition on. This field determines whether to enable context awareness for this
|
|
75
|
+
# workflow's evaluations. Defaults to false.
|
|
76
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
77
|
+
attr_reader :context_awareness
|
|
78
|
+
|
|
79
|
+
sig { params(context_awareness: T::Boolean).void }
|
|
80
|
+
attr_writer :context_awareness
|
|
81
|
+
|
|
70
82
|
# Mapping of guardrail metrics to floating point threshold values. Possible
|
|
71
83
|
# metrics are `correctness`, `completeness`, `instruction_adherence`,
|
|
72
84
|
# `context_adherence`, `ground_truth_adherence`, or `comprehensive_safety`.
|
|
@@ -122,6 +134,7 @@ module Deeprails
|
|
|
122
134
|
Symbol,
|
|
123
135
|
Deeprails::DefendCreateWorkflowParams::AutomaticHallucinationToleranceLevel::OrSymbol
|
|
124
136
|
],
|
|
137
|
+
context_awareness: T::Boolean,
|
|
125
138
|
custom_hallucination_threshold_values: T::Hash[Symbol, Float],
|
|
126
139
|
description: String,
|
|
127
140
|
file_search: T::Array[String],
|
|
@@ -150,6 +163,13 @@ module Deeprails
|
|
|
150
163
|
# `instruction_adherence`, `context_adherence`, `ground_truth_adherence`, or
|
|
151
164
|
# `comprehensive_safety`.
|
|
152
165
|
automatic_hallucination_tolerance_levels: nil,
|
|
166
|
+
# Context includes any structured information that directly relates to the model’s
|
|
167
|
+
# input and expected output—e.g., the recent turn-by-turn history between an AI
|
|
168
|
+
# tutor and a student, facts or state passed through an agentic workflow, or other
|
|
169
|
+
# domain-specific signals your system already knows and wants the model to
|
|
170
|
+
# condition on. This field determines whether to enable context awareness for this
|
|
171
|
+
# workflow's evaluations. Defaults to false.
|
|
172
|
+
context_awareness: nil,
|
|
153
173
|
# Mapping of guardrail metrics to floating point threshold values. Possible
|
|
154
174
|
# metrics are `correctness`, `completeness`, `instruction_adherence`,
|
|
155
175
|
# `context_adherence`, `ground_truth_adherence`, or `comprehensive_safety`.
|
|
@@ -181,6 +201,7 @@ module Deeprails
|
|
|
181
201
|
Symbol,
|
|
182
202
|
Deeprails::DefendCreateWorkflowParams::AutomaticHallucinationToleranceLevel::OrSymbol
|
|
183
203
|
],
|
|
204
|
+
context_awareness: T::Boolean,
|
|
184
205
|
custom_hallucination_threshold_values: T::Hash[Symbol, Float],
|
|
185
206
|
description: String,
|
|
186
207
|
file_search: T::Array[String],
|
|
@@ -20,8 +20,8 @@ module Deeprails
|
|
|
20
20
|
end
|
|
21
21
|
attr_accessor :automatic_hallucination_tolerance_levels
|
|
22
22
|
|
|
23
|
-
# Extended AI capabilities available to the event, if any. Can be `web_search
|
|
24
|
-
# and/or `
|
|
23
|
+
# Extended AI capabilities available to the event, if any. Can be `web_search`,
|
|
24
|
+
# `file_search`, and/or `context_awareness`.
|
|
25
25
|
sig { returns(T::Array[Deeprails::DefendResponse::Capability]) }
|
|
26
26
|
attr_accessor :capabilities
|
|
27
27
|
|
|
@@ -65,7 +65,7 @@ module Deeprails
|
|
|
65
65
|
sig { returns(Time) }
|
|
66
66
|
attr_accessor :updated_at
|
|
67
67
|
|
|
68
|
-
# A unique workflow ID.
|
|
68
|
+
# A unique workflow ID used to identify the workflow in other endpoints.
|
|
69
69
|
sig { returns(String) }
|
|
70
70
|
attr_accessor :workflow_id
|
|
71
71
|
|
|
@@ -119,8 +119,8 @@ module Deeprails
|
|
|
119
119
|
# Mapping of guardrail metric names to tolerance values. Values can be strings
|
|
120
120
|
# (`low`, `medium`, `high`) for automatic tolerance levels.
|
|
121
121
|
automatic_hallucination_tolerance_levels:,
|
|
122
|
-
# Extended AI capabilities available to the event, if any. Can be `web_search
|
|
123
|
-
# and/or `
|
|
122
|
+
# Extended AI capabilities available to the event, if any. Can be `web_search`,
|
|
123
|
+
# `file_search`, and/or `context_awareness`.
|
|
124
124
|
capabilities:,
|
|
125
125
|
# The time the workflow was created in UTC.
|
|
126
126
|
created_at:,
|
|
@@ -144,7 +144,7 @@ module Deeprails
|
|
|
144
144
|
threshold_type:,
|
|
145
145
|
# The most recent time the workflow was updated in UTC.
|
|
146
146
|
updated_at:,
|
|
147
|
-
# A unique workflow ID.
|
|
147
|
+
# A unique workflow ID used to identify the workflow in other endpoints.
|
|
148
148
|
workflow_id:,
|
|
149
149
|
# The action used to improve outputs that fail one or more guardrail metrics for
|
|
150
150
|
# the workflow events.
|
|
@@ -254,6 +254,13 @@ module Deeprails
|
|
|
254
254
|
)
|
|
255
255
|
end
|
|
256
256
|
|
|
257
|
+
# The ID of the billing request for the event.
|
|
258
|
+
sig { returns(T.nilable(String)) }
|
|
259
|
+
attr_reader :billing_request_id
|
|
260
|
+
|
|
261
|
+
sig { params(billing_request_id: String).void }
|
|
262
|
+
attr_writer :billing_request_id
|
|
263
|
+
|
|
257
264
|
# An array of evaluations for this event.
|
|
258
265
|
sig do
|
|
259
266
|
returns(
|
|
@@ -284,42 +291,88 @@ module Deeprails
|
|
|
284
291
|
sig { params(improved_model_output: String).void }
|
|
285
292
|
attr_writer :improved_model_output
|
|
286
293
|
|
|
287
|
-
# Status of the improvement tool used to improve the event.
|
|
288
|
-
|
|
294
|
+
# Status of the improvement tool used to improve the event. `improvement_required`
|
|
295
|
+
# indicates that the evaluation is complete and the improvement action is needed
|
|
296
|
+
# but is not taking place. `improved` and `improvement_failed` indicate when the
|
|
297
|
+
# improvement action concludes, successfully and unsuccessfully, respectively.
|
|
298
|
+
# `no_improvement_required` means that the first evaluation passed all its
|
|
299
|
+
# metrics!
|
|
300
|
+
sig do
|
|
301
|
+
returns(
|
|
302
|
+
T.nilable(
|
|
303
|
+
Deeprails::DefendResponse::Event::ImprovementToolStatus::TaggedSymbol
|
|
304
|
+
)
|
|
305
|
+
)
|
|
306
|
+
end
|
|
289
307
|
attr_reader :improvement_tool_status
|
|
290
308
|
|
|
291
|
-
sig
|
|
309
|
+
sig do
|
|
310
|
+
params(
|
|
311
|
+
improvement_tool_status:
|
|
312
|
+
Deeprails::DefendResponse::Event::ImprovementToolStatus::OrSymbol
|
|
313
|
+
).void
|
|
314
|
+
end
|
|
292
315
|
attr_writer :improvement_tool_status
|
|
293
316
|
|
|
317
|
+
# Status of the event.
|
|
318
|
+
sig do
|
|
319
|
+
returns(
|
|
320
|
+
T.nilable(Deeprails::DefendResponse::Event::Status::TaggedSymbol)
|
|
321
|
+
)
|
|
322
|
+
end
|
|
323
|
+
attr_reader :status
|
|
324
|
+
|
|
325
|
+
sig do
|
|
326
|
+
params(
|
|
327
|
+
status: Deeprails::DefendResponse::Event::Status::OrSymbol
|
|
328
|
+
).void
|
|
329
|
+
end
|
|
330
|
+
attr_writer :status
|
|
331
|
+
|
|
294
332
|
sig do
|
|
295
333
|
params(
|
|
334
|
+
billing_request_id: String,
|
|
296
335
|
evaluations:
|
|
297
336
|
T::Array[Deeprails::DefendResponse::Event::Evaluation::OrHash],
|
|
298
337
|
event_id: String,
|
|
299
338
|
improved_model_output: String,
|
|
300
|
-
improvement_tool_status:
|
|
339
|
+
improvement_tool_status:
|
|
340
|
+
Deeprails::DefendResponse::Event::ImprovementToolStatus::OrSymbol,
|
|
341
|
+
status: Deeprails::DefendResponse::Event::Status::OrSymbol
|
|
301
342
|
).returns(T.attached_class)
|
|
302
343
|
end
|
|
303
344
|
def self.new(
|
|
345
|
+
# The ID of the billing request for the event.
|
|
346
|
+
billing_request_id: nil,
|
|
304
347
|
# An array of evaluations for this event.
|
|
305
348
|
evaluations: nil,
|
|
306
349
|
# A unique workflow event ID.
|
|
307
350
|
event_id: nil,
|
|
308
351
|
# Improved model output after improvement tool was applied.
|
|
309
352
|
improved_model_output: nil,
|
|
310
|
-
# Status of the improvement tool used to improve the event.
|
|
311
|
-
|
|
353
|
+
# Status of the improvement tool used to improve the event. `improvement_required`
|
|
354
|
+
# indicates that the evaluation is complete and the improvement action is needed
|
|
355
|
+
# but is not taking place. `improved` and `improvement_failed` indicate when the
|
|
356
|
+
# improvement action concludes, successfully and unsuccessfully, respectively.
|
|
357
|
+
# `no_improvement_required` means that the first evaluation passed all its
|
|
358
|
+
# metrics!
|
|
359
|
+
improvement_tool_status: nil,
|
|
360
|
+
# Status of the event.
|
|
361
|
+
status: nil
|
|
312
362
|
)
|
|
313
363
|
end
|
|
314
364
|
|
|
315
365
|
sig do
|
|
316
366
|
override.returns(
|
|
317
367
|
{
|
|
368
|
+
billing_request_id: String,
|
|
318
369
|
evaluations:
|
|
319
370
|
T::Array[Deeprails::DefendResponse::Event::Evaluation],
|
|
320
371
|
event_id: String,
|
|
321
372
|
improved_model_output: String,
|
|
322
|
-
improvement_tool_status:
|
|
373
|
+
improvement_tool_status:
|
|
374
|
+
Deeprails::DefendResponse::Event::ImprovementToolStatus::TaggedSymbol,
|
|
375
|
+
status: Deeprails::DefendResponse::Event::Status::TaggedSymbol
|
|
323
376
|
}
|
|
324
377
|
)
|
|
325
378
|
end
|
|
@@ -335,6 +388,14 @@ module Deeprails
|
|
|
335
388
|
)
|
|
336
389
|
end
|
|
337
390
|
|
|
391
|
+
# Analysis of the failures of the model_output according to the guardrail metrics
|
|
392
|
+
# evaluated.
|
|
393
|
+
sig { returns(T.nilable(String)) }
|
|
394
|
+
attr_reader :analysis_of_failures
|
|
395
|
+
|
|
396
|
+
sig { params(analysis_of_failures: String).void }
|
|
397
|
+
attr_writer :analysis_of_failures
|
|
398
|
+
|
|
338
399
|
# The attempt number or identifier for this evaluation.
|
|
339
400
|
sig { returns(T.nilable(String)) }
|
|
340
401
|
attr_reader :attempt
|
|
@@ -384,6 +445,36 @@ module Deeprails
|
|
|
384
445
|
sig { params(guardrail_metrics: T::Array[String]).void }
|
|
385
446
|
attr_writer :guardrail_metrics
|
|
386
447
|
|
|
448
|
+
# Status of the improvement tool used to improve the event. `improvement_required`
|
|
449
|
+
# indicates that the evaluation is complete and the improvement action is needed
|
|
450
|
+
# but is not taking place. `improved` and `improvement_failed` indicate when the
|
|
451
|
+
# improvement action concludes, successfully and unsuccessfully, respectively.
|
|
452
|
+
# `no_improvement_required` means that the first evaluation passed all its
|
|
453
|
+
# metrics!
|
|
454
|
+
sig do
|
|
455
|
+
returns(
|
|
456
|
+
T.nilable(
|
|
457
|
+
Deeprails::DefendResponse::Event::Evaluation::ImprovementToolStatus::TaggedSymbol
|
|
458
|
+
)
|
|
459
|
+
)
|
|
460
|
+
end
|
|
461
|
+
attr_reader :improvement_tool_status
|
|
462
|
+
|
|
463
|
+
sig do
|
|
464
|
+
params(
|
|
465
|
+
improvement_tool_status:
|
|
466
|
+
Deeprails::DefendResponse::Event::Evaluation::ImprovementToolStatus::OrSymbol
|
|
467
|
+
).void
|
|
468
|
+
end
|
|
469
|
+
attr_writer :improvement_tool_status
|
|
470
|
+
|
|
471
|
+
# A list of key improvements made to the model_output to address the failures.
|
|
472
|
+
sig { returns(T.nilable(T::Array[String])) }
|
|
473
|
+
attr_reader :key_improvements
|
|
474
|
+
|
|
475
|
+
sig { params(key_improvements: T::Array[String]).void }
|
|
476
|
+
attr_writer :key_improvements
|
|
477
|
+
|
|
387
478
|
# The model input used for the evaluation.
|
|
388
479
|
sig { returns(T.nilable(T::Hash[Symbol, T.anything])) }
|
|
389
480
|
attr_reader :model_input
|
|
@@ -428,6 +519,7 @@ module Deeprails
|
|
|
428
519
|
|
|
429
520
|
sig do
|
|
430
521
|
params(
|
|
522
|
+
analysis_of_failures: String,
|
|
431
523
|
attempt: String,
|
|
432
524
|
created_at: Time,
|
|
433
525
|
error_message: String,
|
|
@@ -435,6 +527,9 @@ module Deeprails
|
|
|
435
527
|
evaluation_status: String,
|
|
436
528
|
evaluation_total_cost: Float,
|
|
437
529
|
guardrail_metrics: T::Array[String],
|
|
530
|
+
improvement_tool_status:
|
|
531
|
+
Deeprails::DefendResponse::Event::Evaluation::ImprovementToolStatus::OrSymbol,
|
|
532
|
+
key_improvements: T::Array[String],
|
|
438
533
|
model_input: T::Hash[Symbol, T.anything],
|
|
439
534
|
model_output: String,
|
|
440
535
|
modified_at: Time,
|
|
@@ -444,6 +539,9 @@ module Deeprails
|
|
|
444
539
|
).returns(T.attached_class)
|
|
445
540
|
end
|
|
446
541
|
def self.new(
|
|
542
|
+
# Analysis of the failures of the model_output according to the guardrail metrics
|
|
543
|
+
# evaluated.
|
|
544
|
+
analysis_of_failures: nil,
|
|
447
545
|
# The attempt number or identifier for this evaluation.
|
|
448
546
|
attempt: nil,
|
|
449
547
|
# The time the evaluation was created in UTC.
|
|
@@ -458,6 +556,15 @@ module Deeprails
|
|
|
458
556
|
evaluation_total_cost: nil,
|
|
459
557
|
# An array of guardrail metrics evaluated.
|
|
460
558
|
guardrail_metrics: nil,
|
|
559
|
+
# Status of the improvement tool used to improve the event. `improvement_required`
|
|
560
|
+
# indicates that the evaluation is complete and the improvement action is needed
|
|
561
|
+
# but is not taking place. `improved` and `improvement_failed` indicate when the
|
|
562
|
+
# improvement action concludes, successfully and unsuccessfully, respectively.
|
|
563
|
+
# `no_improvement_required` means that the first evaluation passed all its
|
|
564
|
+
# metrics!
|
|
565
|
+
improvement_tool_status: nil,
|
|
566
|
+
# A list of key improvements made to the model_output to address the failures.
|
|
567
|
+
key_improvements: nil,
|
|
461
568
|
# The model input used for the evaluation.
|
|
462
569
|
model_input: nil,
|
|
463
570
|
# The model output that was evaluated.
|
|
@@ -476,6 +583,7 @@ module Deeprails
|
|
|
476
583
|
sig do
|
|
477
584
|
override.returns(
|
|
478
585
|
{
|
|
586
|
+
analysis_of_failures: String,
|
|
479
587
|
attempt: String,
|
|
480
588
|
created_at: Time,
|
|
481
589
|
error_message: String,
|
|
@@ -483,6 +591,9 @@ module Deeprails
|
|
|
483
591
|
evaluation_status: String,
|
|
484
592
|
evaluation_total_cost: Float,
|
|
485
593
|
guardrail_metrics: T::Array[String],
|
|
594
|
+
improvement_tool_status:
|
|
595
|
+
Deeprails::DefendResponse::Event::Evaluation::ImprovementToolStatus::TaggedSymbol,
|
|
596
|
+
key_improvements: T::Array[String],
|
|
486
597
|
model_input: T::Hash[Symbol, T.anything],
|
|
487
598
|
model_output: String,
|
|
488
599
|
modified_at: Time,
|
|
@@ -494,6 +605,141 @@ module Deeprails
|
|
|
494
605
|
end
|
|
495
606
|
def to_hash
|
|
496
607
|
end
|
|
608
|
+
|
|
609
|
+
# Status of the improvement tool used to improve the event. `improvement_required`
|
|
610
|
+
# indicates that the evaluation is complete and the improvement action is needed
|
|
611
|
+
# but is not taking place. `improved` and `improvement_failed` indicate when the
|
|
612
|
+
# improvement action concludes, successfully and unsuccessfully, respectively.
|
|
613
|
+
# `no_improvement_required` means that the first evaluation passed all its
|
|
614
|
+
# metrics!
|
|
615
|
+
module ImprovementToolStatus
|
|
616
|
+
extend Deeprails::Internal::Type::Enum
|
|
617
|
+
|
|
618
|
+
TaggedSymbol =
|
|
619
|
+
T.type_alias do
|
|
620
|
+
T.all(
|
|
621
|
+
Symbol,
|
|
622
|
+
Deeprails::DefendResponse::Event::Evaluation::ImprovementToolStatus
|
|
623
|
+
)
|
|
624
|
+
end
|
|
625
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
626
|
+
|
|
627
|
+
IMPROVED =
|
|
628
|
+
T.let(
|
|
629
|
+
:improved,
|
|
630
|
+
Deeprails::DefendResponse::Event::Evaluation::ImprovementToolStatus::TaggedSymbol
|
|
631
|
+
)
|
|
632
|
+
IMPROVEMENT_FAILED =
|
|
633
|
+
T.let(
|
|
634
|
+
:improvement_failed,
|
|
635
|
+
Deeprails::DefendResponse::Event::Evaluation::ImprovementToolStatus::TaggedSymbol
|
|
636
|
+
)
|
|
637
|
+
NO_IMPROVEMENT_REQUIRED =
|
|
638
|
+
T.let(
|
|
639
|
+
:no_improvement_required,
|
|
640
|
+
Deeprails::DefendResponse::Event::Evaluation::ImprovementToolStatus::TaggedSymbol
|
|
641
|
+
)
|
|
642
|
+
IMPROVEMENT_REQUIRED =
|
|
643
|
+
T.let(
|
|
644
|
+
:improvement_required,
|
|
645
|
+
Deeprails::DefendResponse::Event::Evaluation::ImprovementToolStatus::TaggedSymbol
|
|
646
|
+
)
|
|
647
|
+
|
|
648
|
+
sig do
|
|
649
|
+
override.returns(
|
|
650
|
+
T::Array[
|
|
651
|
+
Deeprails::DefendResponse::Event::Evaluation::ImprovementToolStatus::TaggedSymbol
|
|
652
|
+
]
|
|
653
|
+
)
|
|
654
|
+
end
|
|
655
|
+
def self.values
|
|
656
|
+
end
|
|
657
|
+
end
|
|
658
|
+
end
|
|
659
|
+
|
|
660
|
+
# Status of the improvement tool used to improve the event. `improvement_required`
|
|
661
|
+
# indicates that the evaluation is complete and the improvement action is needed
|
|
662
|
+
# but is not taking place. `improved` and `improvement_failed` indicate when the
|
|
663
|
+
# improvement action concludes, successfully and unsuccessfully, respectively.
|
|
664
|
+
# `no_improvement_required` means that the first evaluation passed all its
|
|
665
|
+
# metrics!
|
|
666
|
+
module ImprovementToolStatus
|
|
667
|
+
extend Deeprails::Internal::Type::Enum
|
|
668
|
+
|
|
669
|
+
TaggedSymbol =
|
|
670
|
+
T.type_alias do
|
|
671
|
+
T.all(
|
|
672
|
+
Symbol,
|
|
673
|
+
Deeprails::DefendResponse::Event::ImprovementToolStatus
|
|
674
|
+
)
|
|
675
|
+
end
|
|
676
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
677
|
+
|
|
678
|
+
IMPROVED =
|
|
679
|
+
T.let(
|
|
680
|
+
:improved,
|
|
681
|
+
Deeprails::DefendResponse::Event::ImprovementToolStatus::TaggedSymbol
|
|
682
|
+
)
|
|
683
|
+
IMPROVEMENT_FAILED =
|
|
684
|
+
T.let(
|
|
685
|
+
:improvement_failed,
|
|
686
|
+
Deeprails::DefendResponse::Event::ImprovementToolStatus::TaggedSymbol
|
|
687
|
+
)
|
|
688
|
+
NO_IMPROVEMENT_REQUIRED =
|
|
689
|
+
T.let(
|
|
690
|
+
:no_improvement_required,
|
|
691
|
+
Deeprails::DefendResponse::Event::ImprovementToolStatus::TaggedSymbol
|
|
692
|
+
)
|
|
693
|
+
IMPROVEMENT_REQUIRED =
|
|
694
|
+
T.let(
|
|
695
|
+
:improvement_required,
|
|
696
|
+
Deeprails::DefendResponse::Event::ImprovementToolStatus::TaggedSymbol
|
|
697
|
+
)
|
|
698
|
+
|
|
699
|
+
sig do
|
|
700
|
+
override.returns(
|
|
701
|
+
T::Array[
|
|
702
|
+
Deeprails::DefendResponse::Event::ImprovementToolStatus::TaggedSymbol
|
|
703
|
+
]
|
|
704
|
+
)
|
|
705
|
+
end
|
|
706
|
+
def self.values
|
|
707
|
+
end
|
|
708
|
+
end
|
|
709
|
+
|
|
710
|
+
# Status of the event.
|
|
711
|
+
module Status
|
|
712
|
+
extend Deeprails::Internal::Type::Enum
|
|
713
|
+
|
|
714
|
+
TaggedSymbol =
|
|
715
|
+
T.type_alias do
|
|
716
|
+
T.all(Symbol, Deeprails::DefendResponse::Event::Status)
|
|
717
|
+
end
|
|
718
|
+
OrSymbol = T.type_alias { T.any(Symbol, String) }
|
|
719
|
+
|
|
720
|
+
COMPLETED =
|
|
721
|
+
T.let(
|
|
722
|
+
:completed,
|
|
723
|
+
Deeprails::DefendResponse::Event::Status::TaggedSymbol
|
|
724
|
+
)
|
|
725
|
+
FAILED =
|
|
726
|
+
T.let(
|
|
727
|
+
:failed,
|
|
728
|
+
Deeprails::DefendResponse::Event::Status::TaggedSymbol
|
|
729
|
+
)
|
|
730
|
+
IN_PROGRESS =
|
|
731
|
+
T.let(
|
|
732
|
+
:in_progress,
|
|
733
|
+
Deeprails::DefendResponse::Event::Status::TaggedSymbol
|
|
734
|
+
)
|
|
735
|
+
|
|
736
|
+
sig do
|
|
737
|
+
override.returns(
|
|
738
|
+
T::Array[Deeprails::DefendResponse::Event::Status::TaggedSymbol]
|
|
739
|
+
)
|
|
740
|
+
end
|
|
741
|
+
def self.values
|
|
742
|
+
end
|
|
497
743
|
end
|
|
498
744
|
end
|
|
499
745
|
|
|
@@ -521,19 +767,45 @@ module Deeprails
|
|
|
521
767
|
sig { params(file_size: Integer).void }
|
|
522
768
|
attr_writer :file_size
|
|
523
769
|
|
|
770
|
+
sig { returns(T.nilable(String)) }
|
|
771
|
+
attr_reader :presigned_url
|
|
772
|
+
|
|
773
|
+
sig { params(presigned_url: String).void }
|
|
774
|
+
attr_writer :presigned_url
|
|
775
|
+
|
|
776
|
+
sig { returns(T.nilable(Time)) }
|
|
777
|
+
attr_reader :presigned_url_expires_at
|
|
778
|
+
|
|
779
|
+
sig { params(presigned_url_expires_at: Time).void }
|
|
780
|
+
attr_writer :presigned_url_expires_at
|
|
781
|
+
|
|
524
782
|
sig do
|
|
525
783
|
params(
|
|
526
784
|
file_id: String,
|
|
527
785
|
file_name: String,
|
|
528
|
-
file_size: Integer
|
|
786
|
+
file_size: Integer,
|
|
787
|
+
presigned_url: String,
|
|
788
|
+
presigned_url_expires_at: Time
|
|
529
789
|
).returns(T.attached_class)
|
|
530
790
|
end
|
|
531
|
-
def self.new(
|
|
791
|
+
def self.new(
|
|
792
|
+
file_id: nil,
|
|
793
|
+
file_name: nil,
|
|
794
|
+
file_size: nil,
|
|
795
|
+
presigned_url: nil,
|
|
796
|
+
presigned_url_expires_at: nil
|
|
797
|
+
)
|
|
532
798
|
end
|
|
533
799
|
|
|
534
800
|
sig do
|
|
535
801
|
override.returns(
|
|
536
|
-
{
|
|
802
|
+
{
|
|
803
|
+
file_id: String,
|
|
804
|
+
file_name: String,
|
|
805
|
+
file_size: Integer,
|
|
806
|
+
presigned_url: String,
|
|
807
|
+
presigned_url_expires_at: Time
|
|
808
|
+
}
|
|
537
809
|
)
|
|
538
810
|
end
|
|
539
811
|
def to_hash
|