deeprails 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +30 -0
- data/README.md +1 -1
- data/lib/deeprails/internal/transport/pooled_net_requester.rb +1 -1
- data/lib/deeprails/internal/util.rb +32 -1
- data/lib/deeprails/models/defend_retrieve_event_params.rb +7 -1
- data/lib/deeprails/models/defend_retrieve_workflow_params.rb +8 -1
- data/lib/deeprails/models/defend_submit_and_stream_event_params.rb +21 -12
- data/lib/deeprails/models/defend_submit_event_params.rb +23 -9
- data/lib/deeprails/models/defend_update_workflow_params.rb +8 -1
- data/lib/deeprails/models/monitor_detail_response.rb +7 -3
- data/lib/deeprails/models/monitor_event_detail_response.rb +5 -3
- data/lib/deeprails/models/monitor_retrieve_event_params.rb +7 -1
- data/lib/deeprails/models/monitor_retrieve_params.rb +8 -1
- data/lib/deeprails/models/monitor_submit_event_params.rb +23 -9
- data/lib/deeprails/models/monitor_update_params.rb +8 -1
- data/lib/deeprails/models/workflow_event_detail_response.rb +14 -3
- data/lib/deeprails/resources/defend.rb +9 -7
- data/lib/deeprails/resources/monitor.rb +2 -1
- data/lib/deeprails/version.rb +1 -1
- data/rbi/deeprails/internal/util.rbi +21 -1
- data/rbi/deeprails/models/defend_retrieve_event_params.rbi +10 -2
- data/rbi/deeprails/models/defend_retrieve_workflow_params.rbi +10 -1
- data/rbi/deeprails/models/defend_submit_and_stream_event_params.rbi +27 -20
- data/rbi/deeprails/models/defend_submit_event_params.rbi +34 -14
- data/rbi/deeprails/models/defend_update_workflow_params.rbi +6 -0
- data/rbi/deeprails/models/monitor_detail_response.rbi +19 -6
- data/rbi/deeprails/models/monitor_event_detail_response.rbi +16 -6
- data/rbi/deeprails/models/monitor_retrieve_event_params.rbi +10 -2
- data/rbi/deeprails/models/monitor_retrieve_params.rbi +10 -1
- data/rbi/deeprails/models/monitor_submit_event_params.rbi +34 -14
- data/rbi/deeprails/models/monitor_update_params.rbi +6 -0
- data/rbi/deeprails/models/workflow_event_detail_response.rbi +37 -3
- data/rbi/deeprails/resources/defend.rbi +13 -8
- data/rbi/deeprails/resources/monitor.rbi +5 -2
- data/sig/deeprails/internal/util.rbs +10 -0
- data/sig/deeprails/models/defend_retrieve_event_params.rbs +6 -1
- data/sig/deeprails/models/defend_retrieve_workflow_params.rbs +6 -1
- data/sig/deeprails/models/defend_submit_and_stream_event_params.rbs +7 -8
- data/sig/deeprails/models/defend_submit_event_params.rbs +16 -5
- data/sig/deeprails/models/defend_update_workflow_params.rbs +5 -0
- data/sig/deeprails/models/monitor_detail_response.rbs +12 -4
- data/sig/deeprails/models/monitor_event_detail_response.rbs +12 -4
- data/sig/deeprails/models/monitor_retrieve_event_params.rbs +6 -1
- data/sig/deeprails/models/monitor_retrieve_params.rbs +6 -1
- data/sig/deeprails/models/monitor_submit_event_params.rbs +16 -5
- data/sig/deeprails/models/monitor_update_params.rbs +5 -0
- data/sig/deeprails/models/workflow_event_detail_response.rbs +16 -4
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 11883d9b5384d524592e17369d91cef2d4eef69b5a3ff4036646115b40fc02b8
|
|
4
|
+
data.tar.gz: 94c8277d39e00367ba6f95d8f7c3d026c96d616f4c95ca50ffeafc190dfdd408
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 1ce43b415bddf92bb9a6ba48a622436d8115b9a2e8beda4e036bb1630efc230e8f8b5edd8e96f171e23ca981c517cb375e88974c7103b46c23ef60f2183abab5
|
|
7
|
+
data.tar.gz: 3bc92a1bc7a37f8bddfd2da84018a14077a10308c85277cb2741a143a4ba316b648a713691bc4fe0e6e7920f35b94cb1816025dbbaf5c624b81f51253d5a937c
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,35 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.24.0 (2026-03-25)
|
|
4
|
+
|
|
5
|
+
Full Changelog: [v0.23.0...v0.24.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.23.0...v0.24.0)
|
|
6
|
+
|
|
7
|
+
### Features
|
|
8
|
+
|
|
9
|
+
* **api:** manual updates ([5ef9233](https://github.com/deeprails/deeprails-ruby-sdk/commit/5ef9233307f25347f1fe5cb31d919b775e86018c))
|
|
10
|
+
* **api:** update run mode lists ([4dd6b35](https://github.com/deeprails/deeprails-ruby-sdk/commit/4dd6b35cfd44db08d215bcffcd82e0df013fdbc1))
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
### Bug Fixes
|
|
14
|
+
|
|
15
|
+
* **client:** always add content-length to post body, even when empty ([a13ed48](https://github.com/deeprails/deeprails-ruby-sdk/commit/a13ed487ebe8d61e23a91427b59568098fa9c509))
|
|
16
|
+
* **client:** loosen json header parsing ([5b586e4](https://github.com/deeprails/deeprails-ruby-sdk/commit/5b586e42540c7495a5a7a90b046f97bd80937a6a))
|
|
17
|
+
* properly mock time in ruby ci tests ([3d4a050](https://github.com/deeprails/deeprails-ruby-sdk/commit/3d4a05086397ebee972f3731c32c546a8680c359))
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
### Chores
|
|
21
|
+
|
|
22
|
+
* **ci:** add build step ([214a42f](https://github.com/deeprails/deeprails-ruby-sdk/commit/214a42f734b006bc8b121c1e7024e48978d0435c))
|
|
23
|
+
* **ci:** skip uploading artifacts on stainless-internal branches ([31cf2d4](https://github.com/deeprails/deeprails-ruby-sdk/commit/31cf2d4fc4f7cf22594abb421e85fc596bfebfe1))
|
|
24
|
+
* **docs:** remove www prefix ([5809c7c](https://github.com/deeprails/deeprails-ruby-sdk/commit/5809c7c3e4f530b4229837034c41fe991627cebe))
|
|
25
|
+
* **internal:** codegen related update ([963603e](https://github.com/deeprails/deeprails-ruby-sdk/commit/963603e86d5835158ef84320a712c4f688369924))
|
|
26
|
+
* **internal:** codegen related update ([729ecae](https://github.com/deeprails/deeprails-ruby-sdk/commit/729ecaebf5923f1c8fb3534af065422d57c93489))
|
|
27
|
+
* **internal:** codegen related update ([5d6fdf8](https://github.com/deeprails/deeprails-ruby-sdk/commit/5d6fdf835fb9189c925b6a4b7e6ab5be0bb5f6c6))
|
|
28
|
+
* **internal:** codegen related update ([3c60669](https://github.com/deeprails/deeprails-ruby-sdk/commit/3c606699d83fa50f2e89502c829bd2a3270848dc))
|
|
29
|
+
* **internal:** remove mock server code ([6da3225](https://github.com/deeprails/deeprails-ruby-sdk/commit/6da3225fe66a6f2e1519b3a5eba404483e0c7092))
|
|
30
|
+
* **internal:** tweak CI branches ([ce92004](https://github.com/deeprails/deeprails-ruby-sdk/commit/ce920042eb923f09c87bfb8d7c0b92e79cc817fd))
|
|
31
|
+
* update mock server docs ([d247ade](https://github.com/deeprails/deeprails-ruby-sdk/commit/d247ade6db1f06e5d56a815cac033b0c5ed7e3c7))
|
|
32
|
+
|
|
3
33
|
## 0.23.0 (2026-01-20)
|
|
4
34
|
|
|
5
35
|
Full Changelog: [v0.22.0...v0.23.0](https://github.com/deeprails/deeprails-ruby-sdk/compare/v0.22.0...v0.23.0)
|
data/README.md
CHANGED
|
@@ -75,7 +75,7 @@ module Deeprails
|
|
|
75
75
|
|
|
76
76
|
case body
|
|
77
77
|
in nil
|
|
78
|
-
|
|
78
|
+
req["content-length"] ||= 0 unless req["transfer-encoding"]
|
|
79
79
|
in String
|
|
80
80
|
req["content-length"] ||= body.bytesize.to_s unless req["transfer-encoding"]
|
|
81
81
|
req.body_stream = Deeprails::Internal::Util::ReadIOAdapter.new(body, &blk)
|
|
@@ -485,11 +485,42 @@ module Deeprails
|
|
|
485
485
|
end
|
|
486
486
|
|
|
487
487
|
# @type [Regexp]
|
|
488
|
-
JSON_CONTENT = %r{^application/(?:
|
|
488
|
+
JSON_CONTENT = %r{^application/(?:[a-zA-Z0-9.-]+\+)?json(?!l)}
|
|
489
489
|
# @type [Regexp]
|
|
490
490
|
JSONL_CONTENT = %r{^application/(:?x-(?:n|l)djson)|(:?(?:x-)?jsonl)}
|
|
491
491
|
|
|
492
492
|
class << self
|
|
493
|
+
# @api private
|
|
494
|
+
#
|
|
495
|
+
# @param query [Hash{Symbol=>Object}]
|
|
496
|
+
#
|
|
497
|
+
# @return [Hash{Symbol=>Object}]
|
|
498
|
+
def encode_query_params(query)
|
|
499
|
+
out = {}
|
|
500
|
+
query.each { write_query_param_element!(out, _1, _2) }
|
|
501
|
+
out
|
|
502
|
+
end
|
|
503
|
+
|
|
504
|
+
# @api private
|
|
505
|
+
#
|
|
506
|
+
# @param collection [Hash{Symbol=>Object}]
|
|
507
|
+
# @param key [String]
|
|
508
|
+
# @param element [Object]
|
|
509
|
+
#
|
|
510
|
+
# @return [nil]
|
|
511
|
+
private def write_query_param_element!(collection, key, element)
|
|
512
|
+
case element
|
|
513
|
+
in Hash
|
|
514
|
+
element.each do |name, value|
|
|
515
|
+
write_query_param_element!(collection, "#{key}[#{name}]", value)
|
|
516
|
+
end
|
|
517
|
+
in Array
|
|
518
|
+
collection[key] = element.map(&:to_s).join(",")
|
|
519
|
+
else
|
|
520
|
+
collection[key] = element.to_s
|
|
521
|
+
end
|
|
522
|
+
end
|
|
523
|
+
|
|
493
524
|
# @api private
|
|
494
525
|
#
|
|
495
526
|
# @param y [Enumerator::Yielder]
|
|
@@ -12,8 +12,14 @@ module Deeprails
|
|
|
12
12
|
# @return [String]
|
|
13
13
|
required :workflow_id, String
|
|
14
14
|
|
|
15
|
-
# @!
|
|
15
|
+
# @!attribute event_id
|
|
16
|
+
#
|
|
17
|
+
# @return [String]
|
|
18
|
+
required :event_id, String
|
|
19
|
+
|
|
20
|
+
# @!method initialize(workflow_id:, event_id:, request_options: {})
|
|
16
21
|
# @param workflow_id [String]
|
|
22
|
+
# @param event_id [String]
|
|
17
23
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
18
24
|
end
|
|
19
25
|
end
|
|
@@ -7,6 +7,11 @@ module Deeprails
|
|
|
7
7
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
8
8
|
include Deeprails::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
|
+
# @!attribute workflow_id
|
|
11
|
+
#
|
|
12
|
+
# @return [String]
|
|
13
|
+
required :workflow_id, String
|
|
14
|
+
|
|
10
15
|
# @!attribute limit
|
|
11
16
|
# Limit the number of returned events associated with this workflow. Defaults
|
|
12
17
|
# to 10.
|
|
@@ -14,10 +19,12 @@ module Deeprails
|
|
|
14
19
|
# @return [Integer, nil]
|
|
15
20
|
optional :limit, Integer
|
|
16
21
|
|
|
17
|
-
# @!method initialize(limit: nil, request_options: {})
|
|
22
|
+
# @!method initialize(workflow_id:, limit: nil, request_options: {})
|
|
18
23
|
# Some parameter documentations has been truncated, see
|
|
19
24
|
# {Deeprails::Models::DefendRetrieveWorkflowParams} for more details.
|
|
20
25
|
#
|
|
26
|
+
# @param workflow_id [String]
|
|
27
|
+
#
|
|
21
28
|
# @param limit [Integer] Limit the number of returned events associated with this workflow. Defaults to
|
|
22
29
|
#
|
|
23
30
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
@@ -7,6 +7,11 @@ module Deeprails
|
|
|
7
7
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
8
8
|
include Deeprails::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
|
+
# @!attribute workflow_id
|
|
11
|
+
#
|
|
12
|
+
# @return [String]
|
|
13
|
+
required :workflow_id, String
|
|
14
|
+
|
|
10
15
|
# @!attribute model_input
|
|
11
16
|
# The input provided to the model (e.g., prompt, messages).
|
|
12
17
|
#
|
|
@@ -26,15 +31,17 @@ module Deeprails
|
|
|
26
31
|
required :model_used, String
|
|
27
32
|
|
|
28
33
|
# @!attribute run_mode
|
|
29
|
-
# The evaluation run mode. Streaming
|
|
30
|
-
#
|
|
34
|
+
# The evaluation run mode. Streaming is supported on all run modes except
|
|
35
|
+
# precision_max and precision_max_codex. Note: super_fast does not support Web
|
|
36
|
+
# Search or File Search — if your workflow has these enabled, use a different run
|
|
37
|
+
# mode or disable the capability on the workflow.
|
|
31
38
|
#
|
|
32
39
|
# @return [Symbol, Deeprails::Models::DefendSubmitAndStreamEventParams::RunMode]
|
|
33
40
|
required :run_mode, enum: -> { Deeprails::DefendSubmitAndStreamEventParams::RunMode }
|
|
34
41
|
|
|
35
42
|
# @!attribute stream
|
|
36
|
-
# Enable SSE streaming for real-time token feedback.
|
|
37
|
-
#
|
|
43
|
+
# Enable SSE streaming for real-time token feedback. Supported on all run modes
|
|
44
|
+
# except precision_max and precision_max_codex.
|
|
38
45
|
#
|
|
39
46
|
# @return [Boolean, nil]
|
|
40
47
|
optional :stream, Deeprails::Internal::Type::Boolean
|
|
@@ -45,35 +52,37 @@ module Deeprails
|
|
|
45
52
|
# @return [String, nil]
|
|
46
53
|
optional :nametag, String
|
|
47
54
|
|
|
48
|
-
# @!method initialize(model_input:, model_output:, model_used:, run_mode:, stream: nil, nametag: nil, request_options: {})
|
|
55
|
+
# @!method initialize(workflow_id:, model_input:, model_output:, model_used:, run_mode:, stream: nil, nametag: nil, request_options: {})
|
|
49
56
|
# Some parameter documentations has been truncated, see
|
|
50
57
|
# {Deeprails::Models::DefendSubmitAndStreamEventParams} for more details.
|
|
51
58
|
#
|
|
59
|
+
# @param workflow_id [String]
|
|
60
|
+
#
|
|
52
61
|
# @param model_input [Hash{Symbol=>Object}] The input provided to the model (e.g., prompt, messages).
|
|
53
62
|
#
|
|
54
63
|
# @param model_output [String] The output generated by the model to be evaluated.
|
|
55
64
|
#
|
|
56
65
|
# @param model_used [String] The model that generated the output (e.g., "gpt-4", "claude-3").
|
|
57
66
|
#
|
|
58
|
-
# @param run_mode [Symbol, Deeprails::Models::DefendSubmitAndStreamEventParams::RunMode] The evaluation run mode. Streaming
|
|
59
|
-
# precision\_
|
|
67
|
+
# @param run_mode [Symbol, Deeprails::Models::DefendSubmitAndStreamEventParams::RunMode] The evaluation run mode. Streaming is supported on all run modes except precisio
|
|
60
68
|
#
|
|
61
|
-
# @param stream [Boolean] Enable SSE streaming for real-time token feedback.
|
|
69
|
+
# @param stream [Boolean] Enable SSE streaming for real-time token feedback. Supported on all run modes ex
|
|
62
70
|
#
|
|
63
71
|
# @param nametag [String] Optional tag to identify this event.
|
|
64
72
|
#
|
|
65
73
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
66
74
|
|
|
67
|
-
# The evaluation run mode. Streaming
|
|
68
|
-
#
|
|
75
|
+
# The evaluation run mode. Streaming is supported on all run modes except
|
|
76
|
+
# precision_max and precision_max_codex. Note: super_fast does not support Web
|
|
77
|
+
# Search or File Search — if your workflow has these enabled, use a different run
|
|
78
|
+
# mode or disable the capability on the workflow.
|
|
69
79
|
module RunMode
|
|
70
80
|
extend Deeprails::Internal::Type::Enum
|
|
71
81
|
|
|
82
|
+
SUPER_FAST = :super_fast
|
|
72
83
|
FAST = :fast
|
|
73
84
|
PRECISION = :precision
|
|
74
85
|
PRECISION_CODEX = :precision_codex
|
|
75
|
-
PRECISION_MAX = :precision_max
|
|
76
|
-
PRECISION_MAX_CODEX = :precision_max_codex
|
|
77
86
|
|
|
78
87
|
# @!method self.values
|
|
79
88
|
# @return [Array<Symbol>]
|
|
@@ -7,6 +7,11 @@ module Deeprails
|
|
|
7
7
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
8
8
|
include Deeprails::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
|
+
# @!attribute workflow_id
|
|
11
|
+
#
|
|
12
|
+
# @return [String]
|
|
13
|
+
required :workflow_id, String
|
|
14
|
+
|
|
10
15
|
# @!attribute model_input
|
|
11
16
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
12
17
|
# contain a `user_prompt` field. For the ground_truth_adherence guardrail metric,
|
|
@@ -30,8 +35,11 @@ module Deeprails
|
|
|
30
35
|
# @!attribute run_mode
|
|
31
36
|
# Run mode for the workflow event. The run mode allows the user to optimize for
|
|
32
37
|
# speed, accuracy, and cost by determining which models are used to evaluate the
|
|
33
|
-
# event. Available run modes
|
|
34
|
-
# `precision`, `
|
|
38
|
+
# event. Available run modes (fastest to most thorough): `super_fast`, `fast`,
|
|
39
|
+
# `precision`, `precision_codex`, `precision_max`, and `precision_max_codex`.
|
|
40
|
+
# Defaults to `fast`. Note: `super_fast` does not support Web Search or File
|
|
41
|
+
# Search — if your workflow has these capabilities enabled, use a different run
|
|
42
|
+
# mode or edit the workflow to disable them.
|
|
35
43
|
#
|
|
36
44
|
# @return [Symbol, Deeprails::Models::DefendSubmitEventParams::RunMode]
|
|
37
45
|
required :run_mode, enum: -> { Deeprails::DefendSubmitEventParams::RunMode }
|
|
@@ -42,10 +50,12 @@ module Deeprails
|
|
|
42
50
|
# @return [String, nil]
|
|
43
51
|
optional :nametag, String
|
|
44
52
|
|
|
45
|
-
# @!method initialize(model_input:, model_output:, model_used:, run_mode:, nametag: nil, request_options: {})
|
|
53
|
+
# @!method initialize(workflow_id:, model_input:, model_output:, model_used:, run_mode:, nametag: nil, request_options: {})
|
|
46
54
|
# Some parameter documentations has been truncated, see
|
|
47
55
|
# {Deeprails::Models::DefendSubmitEventParams} for more details.
|
|
48
56
|
#
|
|
57
|
+
# @param workflow_id [String]
|
|
58
|
+
#
|
|
49
59
|
# @param model_input [Deeprails::Models::DefendSubmitEventParams::ModelInput] A dictionary of inputs sent to the LLM to generate output. The dictionary must c
|
|
50
60
|
#
|
|
51
61
|
# @param model_output [String] Output generated by the LLM to be evaluated.
|
|
@@ -126,16 +136,20 @@ module Deeprails
|
|
|
126
136
|
|
|
127
137
|
# Run mode for the workflow event. The run mode allows the user to optimize for
|
|
128
138
|
# speed, accuracy, and cost by determining which models are used to evaluate the
|
|
129
|
-
# event. Available run modes
|
|
130
|
-
# `precision`, `
|
|
139
|
+
# event. Available run modes (fastest to most thorough): `super_fast`, `fast`,
|
|
140
|
+
# `precision`, `precision_codex`, `precision_max`, and `precision_max_codex`.
|
|
141
|
+
# Defaults to `fast`. Note: `super_fast` does not support Web Search or File
|
|
142
|
+
# Search — if your workflow has these capabilities enabled, use a different run
|
|
143
|
+
# mode or edit the workflow to disable them.
|
|
131
144
|
module RunMode
|
|
132
145
|
extend Deeprails::Internal::Type::Enum
|
|
133
146
|
|
|
134
|
-
|
|
135
|
-
|
|
147
|
+
SUPER_FAST = :super_fast
|
|
148
|
+
FAST = :fast
|
|
136
149
|
PRECISION = :precision
|
|
137
|
-
|
|
138
|
-
|
|
150
|
+
PRECISION_CODEX = :precision_codex
|
|
151
|
+
PRECISION_MAX = :precision_max
|
|
152
|
+
PRECISION_MAX_CODEX = :precision_max_codex
|
|
139
153
|
|
|
140
154
|
# @!method self.values
|
|
141
155
|
# @return [Array<Symbol>]
|
|
@@ -7,6 +7,11 @@ module Deeprails
|
|
|
7
7
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
8
8
|
include Deeprails::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
|
+
# @!attribute workflow_id
|
|
11
|
+
#
|
|
12
|
+
# @return [String]
|
|
13
|
+
required :workflow_id, String
|
|
14
|
+
|
|
10
15
|
# @!attribute automatic_hallucination_tolerance_levels
|
|
11
16
|
# New mapping of guardrail metrics to hallucination tolerance levels (either
|
|
12
17
|
# `low`, `medium`, or `high`) to be used when `threshold_type` is set to
|
|
@@ -84,10 +89,12 @@ module Deeprails
|
|
|
84
89
|
# @return [Boolean, nil]
|
|
85
90
|
optional :web_search, Deeprails::Internal::Type::Boolean
|
|
86
91
|
|
|
87
|
-
# @!method initialize(automatic_hallucination_tolerance_levels: nil, context_awareness: nil, custom_hallucination_threshold_values: nil, description: nil, file_search: nil, improvement_action: nil, max_improvement_attempts: nil, name: nil, threshold_type: nil, web_search: nil, request_options: {})
|
|
92
|
+
# @!method initialize(workflow_id:, automatic_hallucination_tolerance_levels: nil, context_awareness: nil, custom_hallucination_threshold_values: nil, description: nil, file_search: nil, improvement_action: nil, max_improvement_attempts: nil, name: nil, threshold_type: nil, web_search: nil, request_options: {})
|
|
88
93
|
# Some parameter documentations has been truncated, see
|
|
89
94
|
# {Deeprails::Models::DefendUpdateWorkflowParams} for more details.
|
|
90
95
|
#
|
|
96
|
+
# @param workflow_id [String]
|
|
97
|
+
#
|
|
91
98
|
# @param automatic_hallucination_tolerance_levels [Hash{Symbol=>Symbol, Deeprails::Models::DefendUpdateWorkflowParams::AutomaticHallucinationToleranceLevel}] New mapping of guardrail metrics to hallucination tolerance levels
|
|
92
99
|
#
|
|
93
100
|
# @param context_awareness [Boolean] Whether to enable context awareness for this workflow's evaluations.
|
|
@@ -131,6 +131,7 @@ module Deeprails
|
|
|
131
131
|
# @!attribute run_mode
|
|
132
132
|
# Run mode for the evaluation. The run mode allows the user to optimize for speed,
|
|
133
133
|
# accuracy, and cost by determining which models are used to evaluate the event.
|
|
134
|
+
# Note: `super_fast` do not support Web Search or File Search capabilities.
|
|
134
135
|
#
|
|
135
136
|
# @return [Symbol, Deeprails::Models::MonitorDetailResponse::Evaluation::RunMode]
|
|
136
137
|
required :run_mode, enum: -> { Deeprails::MonitorDetailResponse::Evaluation::RunMode }
|
|
@@ -294,15 +295,18 @@ module Deeprails
|
|
|
294
295
|
|
|
295
296
|
# Run mode for the evaluation. The run mode allows the user to optimize for speed,
|
|
296
297
|
# accuracy, and cost by determining which models are used to evaluate the event.
|
|
298
|
+
# Note: `super_fast` do not support Web Search or File Search capabilities.
|
|
297
299
|
#
|
|
298
300
|
# @see Deeprails::Models::MonitorDetailResponse::Evaluation#run_mode
|
|
299
301
|
module RunMode
|
|
300
302
|
extend Deeprails::Internal::Type::Enum
|
|
301
303
|
|
|
302
|
-
|
|
304
|
+
SUPER_FAST = :super_fast
|
|
305
|
+
FAST = :fast
|
|
303
306
|
PRECISION = :precision
|
|
304
|
-
|
|
305
|
-
|
|
307
|
+
PRECISION_CODEX = :precision_codex
|
|
308
|
+
PRECISION_MAX = :precision_max
|
|
309
|
+
PRECISION_MAX_CODEX = :precision_max_codex
|
|
306
310
|
|
|
307
311
|
# @!method self.values
|
|
308
312
|
# @return [Array<Symbol>]
|
|
@@ -158,10 +158,12 @@ module Deeprails
|
|
|
158
158
|
module RunMode
|
|
159
159
|
extend Deeprails::Internal::Type::Enum
|
|
160
160
|
|
|
161
|
-
|
|
161
|
+
SUPER_FAST = :super_fast
|
|
162
|
+
FAST = :fast
|
|
162
163
|
PRECISION = :precision
|
|
163
|
-
|
|
164
|
-
|
|
164
|
+
PRECISION_CODEX = :precision_codex
|
|
165
|
+
PRECISION_MAX = :precision_max
|
|
166
|
+
PRECISION_MAX_CODEX = :precision_max_codex
|
|
165
167
|
|
|
166
168
|
# @!method self.values
|
|
167
169
|
# @return [Array<Symbol>]
|
|
@@ -12,8 +12,14 @@ module Deeprails
|
|
|
12
12
|
# @return [String]
|
|
13
13
|
required :monitor_id, String
|
|
14
14
|
|
|
15
|
-
# @!
|
|
15
|
+
# @!attribute event_id
|
|
16
|
+
#
|
|
17
|
+
# @return [String]
|
|
18
|
+
required :event_id, String
|
|
19
|
+
|
|
20
|
+
# @!method initialize(monitor_id:, event_id:, request_options: {})
|
|
16
21
|
# @param monitor_id [String]
|
|
22
|
+
# @param event_id [String]
|
|
17
23
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
18
24
|
end
|
|
19
25
|
end
|
|
@@ -7,6 +7,11 @@ module Deeprails
|
|
|
7
7
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
8
8
|
include Deeprails::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
|
+
# @!attribute monitor_id
|
|
11
|
+
#
|
|
12
|
+
# @return [String]
|
|
13
|
+
required :monitor_id, String
|
|
14
|
+
|
|
10
15
|
# @!attribute limit
|
|
11
16
|
# Limit the number of returned evaluations associated with this monitor. Defaults
|
|
12
17
|
# to 10.
|
|
@@ -14,10 +19,12 @@ module Deeprails
|
|
|
14
19
|
# @return [Integer, nil]
|
|
15
20
|
optional :limit, Integer
|
|
16
21
|
|
|
17
|
-
# @!method initialize(limit: nil, request_options: {})
|
|
22
|
+
# @!method initialize(monitor_id:, limit: nil, request_options: {})
|
|
18
23
|
# Some parameter documentations has been truncated, see
|
|
19
24
|
# {Deeprails::Models::MonitorRetrieveParams} for more details.
|
|
20
25
|
#
|
|
26
|
+
# @param monitor_id [String]
|
|
27
|
+
#
|
|
21
28
|
# @param limit [Integer] Limit the number of returned evaluations associated with this monitor. Defaults
|
|
22
29
|
#
|
|
23
30
|
# @param request_options [Deeprails::RequestOptions, Hash{Symbol=>Object}]
|
|
@@ -7,6 +7,11 @@ module Deeprails
|
|
|
7
7
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
8
8
|
include Deeprails::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
|
+
# @!attribute monitor_id
|
|
11
|
+
#
|
|
12
|
+
# @return [String]
|
|
13
|
+
required :monitor_id, String
|
|
14
|
+
|
|
10
15
|
# @!attribute model_input
|
|
11
16
|
# A dictionary of inputs sent to the LLM to generate output. The dictionary must
|
|
12
17
|
# contain a `user_prompt` field. For ground_truth_adherence guardrail metric,
|
|
@@ -30,16 +35,21 @@ module Deeprails
|
|
|
30
35
|
# @!attribute run_mode
|
|
31
36
|
# Run mode for the monitor event. The run mode allows the user to optimize for
|
|
32
37
|
# speed, accuracy, and cost by determining which models are used to evaluate the
|
|
33
|
-
# event. Available run modes
|
|
34
|
-
# `precision`, `
|
|
38
|
+
# event. Available run modes (fastest to most thorough): `super_fast`, `fast`,
|
|
39
|
+
# `precision`, `precision_codex`, `precision_max`, and `precision_max_codex`.
|
|
40
|
+
# Defaults to `fast`. Note: `super_fast` does not support Web Search or File
|
|
41
|
+
# Search — if your monitor has these capabilities enabled, use a different run
|
|
42
|
+
# mode or edit the monitor to disable them.
|
|
35
43
|
#
|
|
36
44
|
# @return [Symbol, Deeprails::Models::MonitorSubmitEventParams::RunMode, nil]
|
|
37
45
|
optional :run_mode, enum: -> { Deeprails::MonitorSubmitEventParams::RunMode }
|
|
38
46
|
|
|
39
|
-
# @!method initialize(model_input:, model_output:, nametag: nil, run_mode: nil, request_options: {})
|
|
47
|
+
# @!method initialize(monitor_id:, model_input:, model_output:, nametag: nil, run_mode: nil, request_options: {})
|
|
40
48
|
# Some parameter documentations has been truncated, see
|
|
41
49
|
# {Deeprails::Models::MonitorSubmitEventParams} for more details.
|
|
42
50
|
#
|
|
51
|
+
# @param monitor_id [String]
|
|
52
|
+
#
|
|
43
53
|
# @param model_input [Deeprails::Models::MonitorSubmitEventParams::ModelInput] A dictionary of inputs sent to the LLM to generate output. The dictionary must c
|
|
44
54
|
#
|
|
45
55
|
# @param model_output [String] Output generated by the LLM to be evaluated.
|
|
@@ -118,16 +128,20 @@ module Deeprails
|
|
|
118
128
|
|
|
119
129
|
# Run mode for the monitor event. The run mode allows the user to optimize for
|
|
120
130
|
# speed, accuracy, and cost by determining which models are used to evaluate the
|
|
121
|
-
# event. Available run modes
|
|
122
|
-
# `precision`, `
|
|
131
|
+
# event. Available run modes (fastest to most thorough): `super_fast`, `fast`,
|
|
132
|
+
# `precision`, `precision_codex`, `precision_max`, and `precision_max_codex`.
|
|
133
|
+
# Defaults to `fast`. Note: `super_fast` does not support Web Search or File
|
|
134
|
+
# Search — if your monitor has these capabilities enabled, use a different run
|
|
135
|
+
# mode or edit the monitor to disable them.
|
|
123
136
|
module RunMode
|
|
124
137
|
extend Deeprails::Internal::Type::Enum
|
|
125
138
|
|
|
126
|
-
|
|
127
|
-
|
|
139
|
+
SUPER_FAST = :super_fast
|
|
140
|
+
FAST = :fast
|
|
128
141
|
PRECISION = :precision
|
|
129
|
-
|
|
130
|
-
|
|
142
|
+
PRECISION_CODEX = :precision_codex
|
|
143
|
+
PRECISION_MAX = :precision_max
|
|
144
|
+
PRECISION_MAX_CODEX = :precision_max_codex
|
|
131
145
|
|
|
132
146
|
# @!method self.values
|
|
133
147
|
# @return [Array<Symbol>]
|
|
@@ -7,6 +7,11 @@ module Deeprails
|
|
|
7
7
|
extend Deeprails::Internal::Type::RequestParameters::Converter
|
|
8
8
|
include Deeprails::Internal::Type::RequestParameters
|
|
9
9
|
|
|
10
|
+
# @!attribute monitor_id
|
|
11
|
+
#
|
|
12
|
+
# @return [String]
|
|
13
|
+
required :monitor_id, String
|
|
14
|
+
|
|
10
15
|
# @!attribute description
|
|
11
16
|
# New description of the monitor.
|
|
12
17
|
#
|
|
@@ -47,10 +52,12 @@ module Deeprails
|
|
|
47
52
|
# @return [Boolean, nil]
|
|
48
53
|
optional :web_search, Deeprails::Internal::Type::Boolean
|
|
49
54
|
|
|
50
|
-
# @!method initialize(description: nil, file_search: nil, guardrail_metrics: nil, name: nil, status: nil, web_search: nil, request_options: {})
|
|
55
|
+
# @!method initialize(monitor_id:, description: nil, file_search: nil, guardrail_metrics: nil, name: nil, status: nil, web_search: nil, request_options: {})
|
|
51
56
|
# Some parameter documentations has been truncated, see
|
|
52
57
|
# {Deeprails::Models::MonitorUpdateParams} for more details.
|
|
53
58
|
#
|
|
59
|
+
# @param monitor_id [String]
|
|
60
|
+
#
|
|
54
61
|
# @param description [String] New description of the monitor.
|
|
55
62
|
#
|
|
56
63
|
# @param file_search [Array<String>] An array of file IDs to search in the monitor's evaluations. Files must be uploa
|
|
@@ -63,8 +63,9 @@ module Deeprails
|
|
|
63
63
|
|
|
64
64
|
# @!attribute key_improvements
|
|
65
65
|
#
|
|
66
|
-
# @return [Array<
|
|
67
|
-
required :key_improvements,
|
|
66
|
+
# @return [Array<Deeprails::Models::WorkflowEventDetailResponse::KeyImprovement>]
|
|
67
|
+
required :key_improvements,
|
|
68
|
+
-> { Deeprails::Internal::Type::ArrayOf[Deeprails::WorkflowEventDetailResponse::KeyImprovement] }
|
|
68
69
|
|
|
69
70
|
# @!attribute status
|
|
70
71
|
# Status of the event.
|
|
@@ -141,7 +142,7 @@ module Deeprails
|
|
|
141
142
|
#
|
|
142
143
|
# @param improvement_tool_status [Symbol, Deeprails::Models::WorkflowEventDetailResponse::ImprovementToolStatus, nil] Status of the improvement tool used to improve the event. `improvement_required`
|
|
143
144
|
#
|
|
144
|
-
# @param key_improvements [Array<
|
|
145
|
+
# @param key_improvements [Array<Deeprails::Models::WorkflowEventDetailResponse::KeyImprovement>]
|
|
145
146
|
#
|
|
146
147
|
# @param status [Symbol, Deeprails::Models::WorkflowEventDetailResponse::Status] Status of the event.
|
|
147
148
|
#
|
|
@@ -301,6 +302,16 @@ module Deeprails
|
|
|
301
302
|
# @return [Array<Symbol>]
|
|
302
303
|
end
|
|
303
304
|
|
|
305
|
+
class KeyImprovement < Deeprails::Internal::Type::BaseModel
|
|
306
|
+
# @!attribute key_improvement
|
|
307
|
+
#
|
|
308
|
+
# @return [Array<String>, nil]
|
|
309
|
+
optional :key_improvement, Deeprails::Internal::Type::ArrayOf[String]
|
|
310
|
+
|
|
311
|
+
# @!method initialize(key_improvement: nil)
|
|
312
|
+
# @param key_improvement [Array<String>]
|
|
313
|
+
end
|
|
314
|
+
|
|
304
315
|
# Status of the event.
|
|
305
316
|
#
|
|
306
317
|
# @see Deeprails::Models::WorkflowEventDetailResponse#status
|
|
@@ -92,10 +92,11 @@ module Deeprails
|
|
|
92
92
|
# @see Deeprails::Models::DefendRetrieveWorkflowParams
|
|
93
93
|
def retrieve_workflow(workflow_id, params = {})
|
|
94
94
|
parsed, options = Deeprails::DefendRetrieveWorkflowParams.dump_request(params)
|
|
95
|
+
query = Deeprails::Internal::Util.encode_query_params(parsed)
|
|
95
96
|
@client.request(
|
|
96
97
|
method: :get,
|
|
97
98
|
path: ["defend/%1$s", workflow_id],
|
|
98
|
-
query:
|
|
99
|
+
query: query,
|
|
99
100
|
model: Deeprails::DefendResponse,
|
|
100
101
|
options: options
|
|
101
102
|
)
|
|
@@ -104,8 +105,8 @@ module Deeprails
|
|
|
104
105
|
# Some parameter documentations has been truncated, see
|
|
105
106
|
# {Deeprails::Models::DefendSubmitAndStreamEventParams} for more details.
|
|
106
107
|
#
|
|
107
|
-
# Use this endpoint to
|
|
108
|
-
#
|
|
108
|
+
# Use this endpoint to submit a model input and output pair to a workflow for
|
|
109
|
+
# evaluation with streaming responses.
|
|
109
110
|
#
|
|
110
111
|
# @overload submit_and_stream_event_streaming(workflow_id, model_input:, model_output:, model_used:, run_mode:, stream: nil, nametag: nil, request_options: {})
|
|
111
112
|
#
|
|
@@ -117,9 +118,9 @@ module Deeprails
|
|
|
117
118
|
#
|
|
118
119
|
# @param model_used [String] Body param: The model that generated the output (e.g., "gpt-4", "claude-3").
|
|
119
120
|
#
|
|
120
|
-
# @param run_mode [Symbol, Deeprails::Models::DefendSubmitAndStreamEventParams::RunMode] Body param: The evaluation run mode. Streaming
|
|
121
|
+
# @param run_mode [Symbol, Deeprails::Models::DefendSubmitAndStreamEventParams::RunMode] Body param: The evaluation run mode. Streaming is supported on all run modes exc
|
|
121
122
|
#
|
|
122
|
-
# @param stream [Boolean] Query param: Enable SSE streaming for real-time token feedback.
|
|
123
|
+
# @param stream [Boolean] Query param: Enable SSE streaming for real-time token feedback. Supported on all
|
|
123
124
|
#
|
|
124
125
|
# @param nametag [String] Body param: Optional tag to identify this event.
|
|
125
126
|
#
|
|
@@ -129,12 +130,13 @@ module Deeprails
|
|
|
129
130
|
#
|
|
130
131
|
# @see Deeprails::Models::DefendSubmitAndStreamEventParams
|
|
131
132
|
def submit_and_stream_event_streaming(workflow_id, params)
|
|
132
|
-
parsed, options = Deeprails::DefendSubmitAndStreamEventParams.dump_request(params)
|
|
133
133
|
query_params = [:stream]
|
|
134
|
+
parsed, options = Deeprails::DefendSubmitAndStreamEventParams.dump_request(params)
|
|
135
|
+
query = Deeprails::Internal::Util.encode_query_params(parsed.slice(*query_params))
|
|
134
136
|
@client.request(
|
|
135
137
|
method: :post,
|
|
136
138
|
path: ["defend/%1$s/events?stream=true", workflow_id],
|
|
137
|
-
query:
|
|
139
|
+
query: query,
|
|
138
140
|
headers: {"accept" => "text/event-stream"},
|
|
139
141
|
body: parsed.except(*query_params),
|
|
140
142
|
stream: Deeprails::Internal::Stream,
|
|
@@ -58,10 +58,11 @@ module Deeprails
|
|
|
58
58
|
# @see Deeprails::Models::MonitorRetrieveParams
|
|
59
59
|
def retrieve(monitor_id, params = {})
|
|
60
60
|
parsed, options = Deeprails::MonitorRetrieveParams.dump_request(params)
|
|
61
|
+
query = Deeprails::Internal::Util.encode_query_params(parsed)
|
|
61
62
|
@client.request(
|
|
62
63
|
method: :get,
|
|
63
64
|
path: ["monitor/%1$s", monitor_id],
|
|
64
|
-
query:
|
|
65
|
+
query: query,
|
|
65
66
|
model: Deeprails::MonitorDetailResponse,
|
|
66
67
|
options: options
|
|
67
68
|
)
|
data/lib/deeprails/version.rb
CHANGED