deeprails 0.9.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/README.md +23 -1
- data/lib/deeprails/client.rb +4 -0
- data/lib/deeprails/models/defend_create_workflow_params.rb +18 -1
- data/lib/deeprails/models/file_response.rb +49 -0
- data/lib/deeprails/models/file_upload_params.rb +22 -0
- data/lib/deeprails/models/monitor_create_params.rb +47 -1
- data/lib/deeprails/models/monitor_submit_event_params.rb +1 -35
- data/lib/deeprails/models.rb +4 -0
- data/lib/deeprails/resources/defend.rb +5 -1
- data/lib/deeprails/resources/files.rb +37 -0
- data/lib/deeprails/resources/monitor.rb +11 -6
- data/lib/deeprails/version.rb +1 -1
- data/lib/deeprails.rb +3 -0
- data/rbi/deeprails/client.rbi +3 -0
- data/rbi/deeprails/models/defend_create_workflow_params.rbi +24 -0
- data/rbi/deeprails/models/file_response.rbi +84 -0
- data/rbi/deeprails/models/file_upload_params.rbi +43 -0
- data/rbi/deeprails/models/monitor_create_params.rbi +97 -0
- data/rbi/deeprails/models/monitor_submit_event_params.rbi +0 -88
- data/rbi/deeprails/models.rbi +4 -0
- data/rbi/deeprails/resources/defend.rbi +7 -0
- data/rbi/deeprails/resources/files.rbi +26 -0
- data/rbi/deeprails/resources/monitor.rbi +14 -12
- data/sig/deeprails/client.rbs +2 -0
- data/sig/deeprails/models/defend_create_workflow_params.rbs +15 -1
- data/sig/deeprails/models/file_response.rbs +50 -0
- data/sig/deeprails/models/file_upload_params.rbs +24 -0
- data/sig/deeprails/models/monitor_create_params.rbs +44 -1
- data/sig/deeprails/models/monitor_submit_event_params.rbs +0 -33
- data/sig/deeprails/models.rbs +4 -0
- data/sig/deeprails/resources/defend.rbs +2 -0
- data/sig/deeprails/resources/files.rbs +12 -0
- data/sig/deeprails/resources/monitor.rbs +3 -2
- metadata +11 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 27710c0ec0510319e5002b296aa2eecb635640fee1a38423cf43e8792abd1c9a
|
|
4
|
+
data.tar.gz: 372425505cca1a9bfea94c6adbe93c99cf13cfdf26ea4506ae934e0049486acd
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 03e5bbb5226ef22805593349a2ccd1e4f888405bcb15b19eadc01fcd0e001d88a1b096ccfabc467f8b5cd6de6217d5dfc13122911a41b9866fb4467282e9d14f
|
|
7
|
+
data.tar.gz: 2bfd88ef33139e1ac30af2d3d4098e55061f6712736e61598908e972612cf1053eff0ba1cacc052c2a3e8522742617d320ae1fc471000ef9fc4f4749ad121fb8
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,22 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.11.0 (2025-11-02)
|
|
4
|
+
|
|
5
|
+
Full Changelog: [v0.10.0...v0.11.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.10.0...v0.11.0)
|
|
6
|
+
|
|
7
|
+
### Features
|
|
8
|
+
|
|
9
|
+
* **api:** restructure monitor service ([9682519](https://github.com/deeprails/deeprails-ruby-sdk/commit/968251927003819495ee8c2e5ec9c9276f0a298d))
|
|
10
|
+
|
|
11
|
+
## 0.10.0 (2025-10-30)
|
|
12
|
+
|
|
13
|
+
Full Changelog: [v0.9.0...v0.10.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.9.0...v0.10.0)
|
|
14
|
+
|
|
15
|
+
### Features
|
|
16
|
+
|
|
17
|
+
* **api:** Add file and web search as extended ai capabilities ([9c3e153](https://github.com/deeprails/deeprails-ruby-sdk/commit/9c3e153565a92a3e4228ffd1193cff119f7acbfe))
|
|
18
|
+
* **api:** manual updates ([7c6bd65](https://github.com/deeprails/deeprails-ruby-sdk/commit/7c6bd65a0fdafc880b467aa2ab5bc587d300bf76))
|
|
19
|
+
|
|
3
20
|
## 0.9.0 (2025-10-29)
|
|
4
21
|
|
|
5
22
|
Full Changelog: [v0.8.0...v0.9.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.8.0...v0.9.0)
|
data/README.md
CHANGED
|
@@ -15,7 +15,7 @@ To use this gem, install via Bundler by adding the following to your application
|
|
|
15
15
|
<!-- x-release-please-start-version -->
|
|
16
16
|
|
|
17
17
|
```ruby
|
|
18
|
-
gem "deeprails", "~> 0.
|
|
18
|
+
gem "deeprails", "~> 0.11.0"
|
|
19
19
|
```
|
|
20
20
|
|
|
21
21
|
<!-- x-release-please-end -->
|
|
@@ -40,6 +40,28 @@ defend_response = deeprails.defend.create_workflow(
|
|
|
40
40
|
puts(defend_response.workflow_id)
|
|
41
41
|
```
|
|
42
42
|
|
|
43
|
+
### File uploads
|
|
44
|
+
|
|
45
|
+
Request parameters that correspond to file uploads can be passed as raw contents, a [`Pathname`](https://rubyapi.org/3.2/o/pathname) instance, [`StringIO`](https://rubyapi.org/3.2/o/stringio), or more.
|
|
46
|
+
|
|
47
|
+
```ruby
|
|
48
|
+
require "pathname"
|
|
49
|
+
|
|
50
|
+
# Use `Pathname` to send the filename and/or avoid paging a large file into memory:
|
|
51
|
+
file_response = deeprails.files.upload(file: Pathname("/path/to/file"))
|
|
52
|
+
|
|
53
|
+
# Alternatively, pass file contents or a `StringIO` directly:
|
|
54
|
+
file_response = deeprails.files.upload(file: File.read("/path/to/file"))
|
|
55
|
+
|
|
56
|
+
# Or, to control the filename and/or content type:
|
|
57
|
+
file = Deeprails::FilePart.new(File.read("/path/to/file"), filename: "/path/to/file", content_type: "…")
|
|
58
|
+
file_response = deeprails.files.upload(file: file)
|
|
59
|
+
|
|
60
|
+
puts(file_response.file_id)
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
Note that you can also pass a raw `IO` descriptor, but this disables retries, as the library can't be sure if the descriptor is a file or pipe (which cannot be rewound).
|
|
64
|
+
|
|
43
65
|
### Handling errors
|
|
44
66
|
|
|
45
67
|
When the library is unable to connect to the API, or if the API returns a non-success status code (i.e., 4xx or 5xx response), a subclass of `Deeprails::Errors::APIError` will be thrown:
|
data/lib/deeprails/client.rb
CHANGED
|
@@ -24,6 +24,9 @@ module Deeprails
|
|
|
24
24
|
# @return [Deeprails::Resources::Monitor]
|
|
25
25
|
attr_reader :monitor
|
|
26
26
|
|
|
27
|
+
# @return [Deeprails::Resources::Files]
|
|
28
|
+
attr_reader :files
|
|
29
|
+
|
|
27
30
|
# @api private
|
|
28
31
|
#
|
|
29
32
|
# @return [Hash{String=>String}]
|
|
@@ -73,6 +76,7 @@ module Deeprails
|
|
|
73
76
|
|
|
74
77
|
@defend = Deeprails::Resources::Defend.new(client: self)
|
|
75
78
|
@monitor = Deeprails::Resources::Monitor.new(client: self)
|
|
79
|
+
@files = Deeprails::Resources::Files.new(client: self)
|
|
76
80
|
end
|
|
77
81
|
end
|
|
78
82
|
end
|
|
@@ -57,6 +57,13 @@ module Deeprails
|
|
|
57
57
|
# @return [String, nil]
|
|
58
58
|
optional :description, String
|
|
59
59
|
|
|
60
|
+
# @!attribute file_search
|
|
61
|
+
# An array of file IDs to search in the workflow's evaluations. Files must be
|
|
62
|
+
# uploaded via the DeepRails API first.
|
|
63
|
+
#
|
|
64
|
+
# @return [Array<String>, nil]
|
|
65
|
+
optional :file_search, Deeprails::Internal::Type::ArrayOf[String]
|
|
66
|
+
|
|
60
67
|
# @!attribute max_improvement_attempts
|
|
61
68
|
# Max. number of improvement action retries until a given event passes the
|
|
62
69
|
# guardrails. Defaults to 10.
|
|
@@ -64,7 +71,13 @@ module Deeprails
|
|
|
64
71
|
# @return [Integer, nil]
|
|
65
72
|
optional :max_improvement_attempts, Integer
|
|
66
73
|
|
|
67
|
-
# @!
|
|
74
|
+
# @!attribute web_search
|
|
75
|
+
# Whether to enable web search for this workflow's evaluations. Defaults to false.
|
|
76
|
+
#
|
|
77
|
+
# @return [Boolean, nil]
|
|
78
|
+
optional :web_search, Deeprails::Internal::Type::Boolean
|
|
79
|
+
|
|
80
|
+
# @!method initialize(improvement_action:, name:, type:, automatic_hallucination_tolerance_levels: nil, custom_hallucination_threshold_values: nil, description: nil, file_search: nil, max_improvement_attempts: nil, web_search: nil, request_options: {})
|
|
68
81
|
# Some parameter documentations has been truncated, see
|
|
69
82
|
# {Deeprails::Models::DefendCreateWorkflowParams} for more details.
|
|
70
83
|
#
|
|
@@ -80,8 +93,12 @@ module Deeprails
|
|
|
80
93
|
#
|
|
81
94
|
# @param description [String] Description for the workflow.
|
|
82
95
|
#
|
|
96
|
+
# @param file_search [Array<String>] An array of file IDs to search in the workflow's evaluations. Files must be uplo
|
|
97
|
+
#
|
|
83
98
|
# @param max_improvement_attempts [Integer] Max. number of improvement action retries until a given event passes the guardra
|
|
84
99
|
#
|
|
100
|
+
# @param web_search [Boolean] Whether to enable web search for this workflow's evaluations. Defaults to false.
|
|
101
|
+
#
|
|
85
102
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
86
103
|
|
|
87
104
|
# The action used to improve outputs that fail one or guardrail metrics for the
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Deeprails
|
|
4
|
+
module Models
|
|
5
|
+
# @see Deeprails::Resources::Files#upload
|
|
6
|
+
class FileResponse < Deeprails::Internal::Type::BaseModel
|
|
7
|
+
# @!attribute created_at
|
|
8
|
+
# The time the file was created in UTC.
|
|
9
|
+
#
|
|
10
|
+
# @return [Time, nil]
|
|
11
|
+
optional :created_at, Time
|
|
12
|
+
|
|
13
|
+
# @!attribute file_id
|
|
14
|
+
# A unique file ID.
|
|
15
|
+
#
|
|
16
|
+
# @return [String, nil]
|
|
17
|
+
optional :file_id, String
|
|
18
|
+
|
|
19
|
+
# @!attribute file_name
|
|
20
|
+
# Name of the file.
|
|
21
|
+
#
|
|
22
|
+
# @return [String, nil]
|
|
23
|
+
optional :file_name, String
|
|
24
|
+
|
|
25
|
+
# @!attribute file_path
|
|
26
|
+
# Path to the s3 bucket where the file is stored.
|
|
27
|
+
#
|
|
28
|
+
# @return [String, nil]
|
|
29
|
+
optional :file_path, String
|
|
30
|
+
|
|
31
|
+
# @!attribute updated_at
|
|
32
|
+
# The most recent time the file was modified in UTC.
|
|
33
|
+
#
|
|
34
|
+
# @return [Time, nil]
|
|
35
|
+
optional :updated_at, Time
|
|
36
|
+
|
|
37
|
+
# @!method initialize(created_at: nil, file_id: nil, file_name: nil, file_path: nil, updated_at: nil)
|
|
38
|
+
# @param created_at [Time] The time the file was created in UTC.
|
|
39
|
+
#
|
|
40
|
+
# @param file_id [String] A unique file ID.
|
|
41
|
+
#
|
|
42
|
+
# @param file_name [String] Name of the file.
|
|
43
|
+
#
|
|
44
|
+
# @param file_path [String] Path to the s3 bucket where the file is stored.
|
|
45
|
+
#
|
|
46
|
+
# @param updated_at [Time] The most recent time the file was modified in UTC.
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Deeprails
|
|
4
|
+
module Models
|
|
5
|
+
# @see Deeprails::Resources::Files#upload
|
|
6
|
+
class FileUploadParams < Deeprails::Internal::Type::BaseModel
|
|
7
|
+
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
8
|
+
include Deeprails::Internal::Type::RequestParameters
|
|
9
|
+
|
|
10
|
+
# @!attribute file
|
|
11
|
+
# The contents of the file to upload.
|
|
12
|
+
#
|
|
13
|
+
# @return [Pathname, StringIO, IO, String, Deeprails::FilePart]
|
|
14
|
+
required :file, Deeprails::Internal::Type::FileInput
|
|
15
|
+
|
|
16
|
+
# @!method initialize(file:, request_options: {})
|
|
17
|
+
# @param file [Pathname, StringIO, IO, String, Deeprails::FilePart] The contents of the file to upload.
|
|
18
|
+
#
|
|
19
|
+
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -7,6 +7,16 @@ module Deeprails
|
|
|
7
7
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
8
8
|
include Deeprails::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
|
+
# @!attribute guardrail_metrics
|
|
11
|
+
# An array of guardrail metrics that the model input and output pair will be
|
|
12
|
+
# evaluated on. For non-enterprise users, these will be limited to `correctness`,
|
|
13
|
+
# `completeness`, `instruction_adherence`, `context_adherence`,
|
|
14
|
+
# `ground_truth_adherence`, and/or `comprehensive_safety`.
|
|
15
|
+
#
|
|
16
|
+
# @return [Array<Symbol, Deeprails::Models::MonitorCreateParams::GuardrailMetric>]
|
|
17
|
+
required :guardrail_metrics,
|
|
18
|
+
-> { Deeprails::Internal::Type::ArrayOf[enum: Deeprails::MonitorCreateParams::GuardrailMetric] }
|
|
19
|
+
|
|
10
20
|
# @!attribute name
|
|
11
21
|
# Name of the new monitor.
|
|
12
22
|
#
|
|
@@ -19,12 +29,48 @@ module Deeprails
|
|
|
19
29
|
# @return [String, nil]
|
|
20
30
|
optional :description, String
|
|
21
31
|
|
|
22
|
-
# @!
|
|
32
|
+
# @!attribute file_search
|
|
33
|
+
# An array of file IDs to search in the monitor's evaluations. Files must be
|
|
34
|
+
# uploaded via the DeepRails API first.
|
|
35
|
+
#
|
|
36
|
+
# @return [Array<String>, nil]
|
|
37
|
+
optional :file_search, Deeprails::Internal::Type::ArrayOf[String]
|
|
38
|
+
|
|
39
|
+
# @!attribute web_search
|
|
40
|
+
# Whether to enable web search for this monitor's evaluations. Defaults to false.
|
|
41
|
+
#
|
|
42
|
+
# @return [Boolean, nil]
|
|
43
|
+
optional :web_search, Deeprails::Internal::Type::Boolean
|
|
44
|
+
|
|
45
|
+
# @!method initialize(guardrail_metrics:, name:, description: nil, file_search: nil, web_search: nil, request_options: {})
|
|
46
|
+
# Some parameter documentations has been truncated, see
|
|
47
|
+
# {Deeprails::Models::MonitorCreateParams} for more details.
|
|
48
|
+
#
|
|
49
|
+
# @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorCreateParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
|
|
50
|
+
#
|
|
23
51
|
# @param name [String] Name of the new monitor.
|
|
24
52
|
#
|
|
25
53
|
# @param description [String] Description of the new monitor.
|
|
26
54
|
#
|
|
55
|
+
# @param file_search [Array<String>] An array of file IDs to search in the monitor's evaluations. Files must be uploa
|
|
56
|
+
#
|
|
57
|
+
# @param web_search [Boolean] Whether to enable web search for this monitor's evaluations. Defaults to false.
|
|
58
|
+
#
|
|
27
59
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
60
|
+
|
|
61
|
+
module GuardrailMetric
|
|
62
|
+
extend Deeprails::Internal::Type::Enum
|
|
63
|
+
|
|
64
|
+
CORRECTNESS = :correctness
|
|
65
|
+
COMPLETENESS = :completeness
|
|
66
|
+
INSTRUCTION_ADHERENCE = :instruction_adherence
|
|
67
|
+
CONTEXT_ADHERENCE = :context_adherence
|
|
68
|
+
GROUND_TRUTH_ADHERENCE = :ground_truth_adherence
|
|
69
|
+
COMPREHENSIVE_SAFETY = :comprehensive_safety
|
|
70
|
+
|
|
71
|
+
# @!method self.values
|
|
72
|
+
# @return [Array<Symbol>]
|
|
73
|
+
end
|
|
28
74
|
end
|
|
29
75
|
end
|
|
30
76
|
end
|
|
@@ -7,16 +7,6 @@ module Deeprails
|
|
|
7
7
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
8
8
|
include Deeprails::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
|
-
# @!attribute guardrail_metrics
|
|
11
|
-
# An array of guardrail metrics that the model input and output pair will be
|
|
12
|
-
# evaluated on. For non-enterprise users, these will be limited to `correctness`,
|
|
13
|
-
# `completeness`, `instruction_adherence`, `context_adherence`,
|
|
14
|
-
# `ground_truth_adherence`, and/or `comprehensive_safety`.
|
|
15
|
-
#
|
|
16
|
-
# @return [Array<Symbol, Deeprails::Models::MonitorSubmitEventParams::GuardrailMetric>]
|
|
17
|
-
required :guardrail_metrics,
|
|
18
|
-
-> { Deeprails::Internal::Type::ArrayOf[enum: Deeprails::MonitorSubmitEventParams::GuardrailMetric] }
|
|
19
|
-
|
|
20
10
|
# @!attribute model_input
|
|
21
11
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
22
12
|
# contain at least a `user_prompt` field or a `system_prompt` field. For
|
|
@@ -31,12 +21,6 @@ module Deeprails
|
|
|
31
21
|
# @return [String]
|
|
32
22
|
required :model_output, String
|
|
33
23
|
|
|
34
|
-
# @!attribute model_used
|
|
35
|
-
# Model ID used to generate the output, like `gpt-4o` or `o3`.
|
|
36
|
-
#
|
|
37
|
-
# @return [String, nil]
|
|
38
|
-
optional :model_used, String
|
|
39
|
-
|
|
40
24
|
# @!attribute nametag
|
|
41
25
|
# An optional, user-defined tag for the event.
|
|
42
26
|
#
|
|
@@ -52,38 +36,20 @@ module Deeprails
|
|
|
52
36
|
# @return [Symbol, Deeprails::Models::MonitorSubmitEventParams::RunMode, nil]
|
|
53
37
|
optional :run_mode, enum: -> { Deeprails::MonitorSubmitEventParams::RunMode }
|
|
54
38
|
|
|
55
|
-
# @!method initialize(
|
|
39
|
+
# @!method initialize(model_input:, model_output:, nametag: nil, run_mode: nil, request_options: {})
|
|
56
40
|
# Some parameter documentations has been truncated, see
|
|
57
41
|
# {Deeprails::Models::MonitorSubmitEventParams} for more details.
|
|
58
42
|
#
|
|
59
|
-
# @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorSubmitEventParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
|
|
60
|
-
#
|
|
61
43
|
# @param model_input [Deeprails::Models::MonitorSubmitEventParams::ModelInput] A dictionary of inputs sent to the LLM to generate output. The dictionary must c
|
|
62
44
|
#
|
|
63
45
|
# @param model_output [String] Output generated by the LLM to be evaluated.
|
|
64
46
|
#
|
|
65
|
-
# @param model_used [String] Model ID used to generate the output, like `gpt-4o` or `o3`.
|
|
66
|
-
#
|
|
67
47
|
# @param nametag [String] An optional, user-defined tag for the event.
|
|
68
48
|
#
|
|
69
49
|
# @param run_mode [Symbol, Deeprails::Models::MonitorSubmitEventParams::RunMode] Run mode for the monitor event. The run mode allows the user to optimize for sp
|
|
70
50
|
#
|
|
71
51
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
72
52
|
|
|
73
|
-
module GuardrailMetric
|
|
74
|
-
extend Deeprails::Internal::Type::Enum
|
|
75
|
-
|
|
76
|
-
CORRECTNESS = :correctness
|
|
77
|
-
COMPLETENESS = :completeness
|
|
78
|
-
INSTRUCTION_ADHERENCE = :instruction_adherence
|
|
79
|
-
CONTEXT_ADHERENCE = :context_adherence
|
|
80
|
-
GROUND_TRUTH_ADHERENCE = :ground_truth_adherence
|
|
81
|
-
COMPREHENSIVE_SAFETY = :comprehensive_safety
|
|
82
|
-
|
|
83
|
-
# @!method self.values
|
|
84
|
-
# @return [Array<Symbol>]
|
|
85
|
-
end
|
|
86
|
-
|
|
87
53
|
class ModelInput < Deeprails::Internal::Type::BaseModel
|
|
88
54
|
# @!attribute ground_truth
|
|
89
55
|
# The ground truth for evaluating Ground Truth Adherence guardrail.
|
data/lib/deeprails/models.rb
CHANGED
|
@@ -51,6 +51,10 @@ module Deeprails
|
|
|
51
51
|
|
|
52
52
|
DefendUpdateWorkflowParams = Deeprails::Models::DefendUpdateWorkflowParams
|
|
53
53
|
|
|
54
|
+
FileResponse = Deeprails::Models::FileResponse
|
|
55
|
+
|
|
56
|
+
FileUploadParams = Deeprails::Models::FileUploadParams
|
|
57
|
+
|
|
54
58
|
MonitorCreateParams = Deeprails::Models::MonitorCreateParams
|
|
55
59
|
|
|
56
60
|
MonitorDetailResponse = Deeprails::Models::MonitorDetailResponse
|
|
@@ -9,7 +9,7 @@ module Deeprails
|
|
|
9
9
|
# Use this endpoint to create a new guardrail workflow with optional guardrail
|
|
10
10
|
# thresholds and improvement actions
|
|
11
11
|
#
|
|
12
|
-
# @overload create_workflow(improvement_action:, name:, type:, automatic_hallucination_tolerance_levels: nil, custom_hallucination_threshold_values: nil, description: nil, max_improvement_attempts: nil, request_options: {})
|
|
12
|
+
# @overload create_workflow(improvement_action:, name:, type:, automatic_hallucination_tolerance_levels: nil, custom_hallucination_threshold_values: nil, description: nil, file_search: nil, max_improvement_attempts: nil, web_search: nil, request_options: {})
|
|
13
13
|
#
|
|
14
14
|
# @param improvement_action [Symbol, Deeprails::Models::DefendCreateWorkflowParams::ImprovementAction] The action used to improve outputs that fail one or guardrail metrics for the wo
|
|
15
15
|
#
|
|
@@ -23,8 +23,12 @@ module Deeprails
|
|
|
23
23
|
#
|
|
24
24
|
# @param description [String] Description for the workflow.
|
|
25
25
|
#
|
|
26
|
+
# @param file_search [Array<String>] An array of file IDs to search in the workflow's evaluations. Files must be uplo
|
|
27
|
+
#
|
|
26
28
|
# @param max_improvement_attempts [Integer] Max. number of improvement action retries until a given event passes the guardra
|
|
27
29
|
#
|
|
30
|
+
# @param web_search [Boolean] Whether to enable web search for this workflow's evaluations. Defaults to false.
|
|
31
|
+
#
|
|
28
32
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
29
33
|
#
|
|
30
34
|
# @return [Deeprails::Models::DefendResponse]
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Deeprails
|
|
4
|
+
module Resources
|
|
5
|
+
class Files
|
|
6
|
+
# Use this endpoint to upload a file to the DeepRails API
|
|
7
|
+
#
|
|
8
|
+
# @overload upload(file:, request_options: {})
|
|
9
|
+
#
|
|
10
|
+
# @param file [Pathname, StringIO, IO, String, Deeprails::FilePart] The contents of the file to upload.
|
|
11
|
+
#
|
|
12
|
+
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
13
|
+
#
|
|
14
|
+
# @return [Deeprails::Models::FileResponse]
|
|
15
|
+
#
|
|
16
|
+
# @see Deeprails::Models::FileUploadParams
|
|
17
|
+
def upload(params)
|
|
18
|
+
parsed, options = Deeprails::FileUploadParams.dump_request(params)
|
|
19
|
+
@client.request(
|
|
20
|
+
method: :post,
|
|
21
|
+
path: "files/upload",
|
|
22
|
+
headers: {"content-type" => "multipart/form-data"},
|
|
23
|
+
body: parsed,
|
|
24
|
+
model: Deeprails::FileResponse,
|
|
25
|
+
options: options
|
|
26
|
+
)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# @api private
|
|
30
|
+
#
|
|
31
|
+
# @param client [Deeprails::Client]
|
|
32
|
+
def initialize(client:)
|
|
33
|
+
@client = client
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -3,15 +3,24 @@
|
|
|
3
3
|
module Deeprails
|
|
4
4
|
module Resources
|
|
5
5
|
class Monitor
|
|
6
|
+
# Some parameter documentations has been truncated, see
|
|
7
|
+
# {Deeprails::Models::MonitorCreateParams} for more details.
|
|
8
|
+
#
|
|
6
9
|
# Use this endpoint to create a new monitor to evaluate model inputs and outputs
|
|
7
10
|
# using guardrails
|
|
8
11
|
#
|
|
9
|
-
# @overload create(name:, description: nil, request_options: {})
|
|
12
|
+
# @overload create(guardrail_metrics:, name:, description: nil, file_search: nil, web_search: nil, request_options: {})
|
|
13
|
+
#
|
|
14
|
+
# @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorCreateParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
|
|
10
15
|
#
|
|
11
16
|
# @param name [String] Name of the new monitor.
|
|
12
17
|
#
|
|
13
18
|
# @param description [String] Description of the new monitor.
|
|
14
19
|
#
|
|
20
|
+
# @param file_search [Array<String>] An array of file IDs to search in the monitor's evaluations. Files must be uploa
|
|
21
|
+
#
|
|
22
|
+
# @param web_search [Boolean] Whether to enable web search for this monitor's evaluations. Defaults to false.
|
|
23
|
+
#
|
|
15
24
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}, nil]
|
|
16
25
|
#
|
|
17
26
|
# @return [Deeprails::Models::MonitorResponse]
|
|
@@ -91,18 +100,14 @@ module Deeprails
|
|
|
91
100
|
# Use this endpoint to submit a model input and output pair to a monitor for
|
|
92
101
|
# evaluation
|
|
93
102
|
#
|
|
94
|
-
# @overload submit_event(monitor_id,
|
|
103
|
+
# @overload submit_event(monitor_id, model_input:, model_output:, nametag: nil, run_mode: nil, request_options: {})
|
|
95
104
|
#
|
|
96
105
|
# @param monitor_id [String] The ID of the monitor associated with this event.
|
|
97
106
|
#
|
|
98
|
-
# @param guardrail_metrics [Array<Symbol, Deeprails::Models::MonitorSubmitEventParams::GuardrailMetric>] An array of guardrail metrics that the model input and output pair will be evalu
|
|
99
|
-
#
|
|
100
107
|
# @param model_input [Deeprails::Models::MonitorSubmitEventParams::ModelInput] A dictionary of inputs sent to the LLM to generate output. The dictionary must c
|
|
101
108
|
#
|
|
102
109
|
# @param model_output [String] Output generated by the LLM to be evaluated.
|
|
103
110
|
#
|
|
104
|
-
# @param model_used [String] Model ID used to generate the output, like `gpt-4o` or `o3`.
|
|
105
|
-
#
|
|
106
111
|
# @param nametag [String] An optional, user-defined tag for the event.
|
|
107
112
|
#
|
|
108
113
|
# @param run_mode [Symbol, Deeprails::Models::MonitorSubmitEventParams::RunMode] Run mode for the monitor event. The run mode allows the user to optimize for sp
|
data/lib/deeprails/version.rb
CHANGED
data/lib/deeprails.rb
CHANGED
|
@@ -56,6 +56,8 @@ require_relative "deeprails/models/defend_retrieve_event_params"
|
|
|
56
56
|
require_relative "deeprails/models/defend_retrieve_workflow_params"
|
|
57
57
|
require_relative "deeprails/models/defend_submit_event_params"
|
|
58
58
|
require_relative "deeprails/models/defend_update_workflow_params"
|
|
59
|
+
require_relative "deeprails/models/file_response"
|
|
60
|
+
require_relative "deeprails/models/file_upload_params"
|
|
59
61
|
require_relative "deeprails/models/monitor_create_params"
|
|
60
62
|
require_relative "deeprails/models/monitor_detail_response"
|
|
61
63
|
require_relative "deeprails/models/monitor_event_response"
|
|
@@ -66,4 +68,5 @@ require_relative "deeprails/models/monitor_update_params"
|
|
|
66
68
|
require_relative "deeprails/models/workflow_event_response"
|
|
67
69
|
require_relative "deeprails/models"
|
|
68
70
|
require_relative "deeprails/resources/defend"
|
|
71
|
+
require_relative "deeprails/resources/files"
|
|
69
72
|
require_relative "deeprails/resources/monitor"
|
data/rbi/deeprails/client.rbi
CHANGED
|
@@ -19,6 +19,9 @@ module Deeprails
|
|
|
19
19
|
sig { returns(Deeprails::Resources::Monitor) }
|
|
20
20
|
attr_reader :monitor
|
|
21
21
|
|
|
22
|
+
sig { returns(Deeprails::Resources::Files) }
|
|
23
|
+
attr_reader :files
|
|
24
|
+
|
|
22
25
|
# @api private
|
|
23
26
|
sig { override.returns(T::Hash[String, String]) }
|
|
24
27
|
private def auth_headers
|
|
@@ -85,6 +85,14 @@ module Deeprails
|
|
|
85
85
|
sig { params(description: String).void }
|
|
86
86
|
attr_writer :description
|
|
87
87
|
|
|
88
|
+
# An array of file IDs to search in the workflow's evaluations. Files must be
|
|
89
|
+
# uploaded via the DeepRails API first.
|
|
90
|
+
sig { returns(T.nilable(T::Array[String])) }
|
|
91
|
+
attr_reader :file_search
|
|
92
|
+
|
|
93
|
+
sig { params(file_search: T::Array[String]).void }
|
|
94
|
+
attr_writer :file_search
|
|
95
|
+
|
|
88
96
|
# Max. number of improvement action retries until a given event passes the
|
|
89
97
|
# guardrails. Defaults to 10.
|
|
90
98
|
sig { returns(T.nilable(Integer)) }
|
|
@@ -93,6 +101,13 @@ module Deeprails
|
|
|
93
101
|
sig { params(max_improvement_attempts: Integer).void }
|
|
94
102
|
attr_writer :max_improvement_attempts
|
|
95
103
|
|
|
104
|
+
# Whether to enable web search for this workflow's evaluations. Defaults to false.
|
|
105
|
+
sig { returns(T.nilable(T::Boolean)) }
|
|
106
|
+
attr_reader :web_search
|
|
107
|
+
|
|
108
|
+
sig { params(web_search: T::Boolean).void }
|
|
109
|
+
attr_writer :web_search
|
|
110
|
+
|
|
96
111
|
sig do
|
|
97
112
|
params(
|
|
98
113
|
improvement_action:
|
|
@@ -106,7 +121,9 @@ module Deeprails
|
|
|
106
121
|
],
|
|
107
122
|
custom_hallucination_threshold_values: T::Hash[Symbol, Float],
|
|
108
123
|
description: String,
|
|
124
|
+
file_search: T::Array[String],
|
|
109
125
|
max_improvement_attempts: Integer,
|
|
126
|
+
web_search: T::Boolean,
|
|
110
127
|
request_options: Deeprails::RequestOptions::OrHash
|
|
111
128
|
).returns(T.attached_class)
|
|
112
129
|
end
|
|
@@ -136,9 +153,14 @@ module Deeprails
|
|
|
136
153
|
custom_hallucination_threshold_values: nil,
|
|
137
154
|
# Description for the workflow.
|
|
138
155
|
description: nil,
|
|
156
|
+
# An array of file IDs to search in the workflow's evaluations. Files must be
|
|
157
|
+
# uploaded via the DeepRails API first.
|
|
158
|
+
file_search: nil,
|
|
139
159
|
# Max. number of improvement action retries until a given event passes the
|
|
140
160
|
# guardrails. Defaults to 10.
|
|
141
161
|
max_improvement_attempts: nil,
|
|
162
|
+
# Whether to enable web search for this workflow's evaluations. Defaults to false.
|
|
163
|
+
web_search: nil,
|
|
142
164
|
request_options: {}
|
|
143
165
|
)
|
|
144
166
|
end
|
|
@@ -157,7 +179,9 @@ module Deeprails
|
|
|
157
179
|
],
|
|
158
180
|
custom_hallucination_threshold_values: T::Hash[Symbol, Float],
|
|
159
181
|
description: String,
|
|
182
|
+
file_search: T::Array[String],
|
|
160
183
|
max_improvement_attempts: Integer,
|
|
184
|
+
web_search: T::Boolean,
|
|
161
185
|
request_options: Deeprails::RequestOptions
|
|
162
186
|
}
|
|
163
187
|
)
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# typed: strong
|
|
2
|
+
|
|
3
|
+
module Deeprails
|
|
4
|
+
module Models
|
|
5
|
+
class FileResponse < Deeprails::Internal::Type::BaseModel
|
|
6
|
+
OrHash =
|
|
7
|
+
T.type_alias do
|
|
8
|
+
T.any(Deeprails::FileResponse, Deeprails::Internal::AnyHash)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
# The time the file was created in UTC.
|
|
12
|
+
sig { returns(T.nilable(Time)) }
|
|
13
|
+
attr_reader :created_at
|
|
14
|
+
|
|
15
|
+
sig { params(created_at: Time).void }
|
|
16
|
+
attr_writer :created_at
|
|
17
|
+
|
|
18
|
+
# A unique file ID.
|
|
19
|
+
sig { returns(T.nilable(String)) }
|
|
20
|
+
attr_reader :file_id
|
|
21
|
+
|
|
22
|
+
sig { params(file_id: String).void }
|
|
23
|
+
attr_writer :file_id
|
|
24
|
+
|
|
25
|
+
# Name of the file.
|
|
26
|
+
sig { returns(T.nilable(String)) }
|
|
27
|
+
attr_reader :file_name
|
|
28
|
+
|
|
29
|
+
sig { params(file_name: String).void }
|
|
30
|
+
attr_writer :file_name
|
|
31
|
+
|
|
32
|
+
# Path to the s3 bucket where the file is stored.
|
|
33
|
+
sig { returns(T.nilable(String)) }
|
|
34
|
+
attr_reader :file_path
|
|
35
|
+
|
|
36
|
+
sig { params(file_path: String).void }
|
|
37
|
+
attr_writer :file_path
|
|
38
|
+
|
|
39
|
+
# The most recent time the file was modified in UTC.
|
|
40
|
+
sig { returns(T.nilable(Time)) }
|
|
41
|
+
attr_reader :updated_at
|
|
42
|
+
|
|
43
|
+
sig { params(updated_at: Time).void }
|
|
44
|
+
attr_writer :updated_at
|
|
45
|
+
|
|
46
|
+
sig do
|
|
47
|
+
params(
|
|
48
|
+
created_at: Time,
|
|
49
|
+
file_id: String,
|
|
50
|
+
file_name: String,
|
|
51
|
+
file_path: String,
|
|
52
|
+
updated_at: Time
|
|
53
|
+
).returns(T.attached_class)
|
|
54
|
+
end
|
|
55
|
+
def self.new(
|
|
56
|
+
# The time the file was created in UTC.
|
|
57
|
+
created_at: nil,
|
|
58
|
+
# A unique file ID.
|
|
59
|
+
file_id: nil,
|
|
60
|
+
# Name of the file.
|
|
61
|
+
file_name: nil,
|
|
62
|
+
# Path to the s3 bucket where the file is stored.
|
|
63
|
+
file_path: nil,
|
|
64
|
+
# The most recent time the file was modified in UTC.
|
|
65
|
+
updated_at: nil
|
|
66
|
+
)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
sig do
|
|
70
|
+
override.returns(
|
|
71
|
+
{
|
|
72
|
+
created_at: Time,
|
|
73
|
+
file_id: String,
|
|
74
|
+
file_name: String,
|
|
75
|
+
file_path: String,
|
|
76
|
+
updated_at: Time
|
|
77
|
+
}
|
|
78
|
+
)
|
|
79
|
+
end
|
|
80
|
+
def to_hash
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|