nitro_intelligence 3.0.2 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +3 -1
- data/docs/README.md +62 -10
- data/lib/nitro_intelligence/assistant.rb +12 -2
- data/lib/nitro_intelligence/assistants.rb +209 -93
- data/lib/nitro_intelligence/client/client.rb +2 -2
- data/lib/nitro_intelligence/client/factory.rb +11 -3
- data/lib/nitro_intelligence/client/handlers/image_handler.rb +3 -1
- data/lib/nitro_intelligence/client/observers/langfuse_observer.rb +24 -5
- data/lib/nitro_intelligence/configuration.rb +2 -0
- data/lib/nitro_intelligence/deprecator.rb +10 -0
- data/lib/nitro_intelligence/langfuse_extension.rb +12 -5
- data/lib/nitro_intelligence/langfuse_tracer_provider.rb +22 -1
- data/lib/nitro_intelligence/pending_review.rb +222 -0
- data/lib/nitro_intelligence/railtie.rb +9 -0
- data/lib/nitro_intelligence/reporter.rb +2 -1
- data/lib/nitro_intelligence/review_decision.rb +34 -0
- data/lib/nitro_intelligence/run_result.rb +129 -0
- data/lib/nitro_intelligence/tool_call_review_interrupt.rb +7 -13
- data/lib/nitro_intelligence/tool_call_review_validator.rb +4 -8
- data/lib/nitro_intelligence/version.rb +1 -1
- data/lib/nitro_intelligence.rb +3 -0
- metadata +6 -1
|
@@ -6,8 +6,10 @@ require "nitro_intelligence/observability/project"
|
|
|
6
6
|
module NitroIntelligence
|
|
7
7
|
module Client
|
|
8
8
|
class Factory
|
|
9
|
-
def initialize(observability_project_slug:)
|
|
9
|
+
def initialize(observability_project_slug:, inference_max_retries: nil, inference_timeout: nil)
|
|
10
10
|
@observability_project_slug = observability_project_slug
|
|
11
|
+
@inference_max_retries = inference_max_retries
|
|
12
|
+
@inference_timeout = inference_timeout
|
|
11
13
|
end
|
|
12
14
|
|
|
13
15
|
def build
|
|
@@ -42,10 +44,16 @@ module NitroIntelligence
|
|
|
42
44
|
project_client
|
|
43
45
|
end
|
|
44
46
|
|
|
47
|
+
# The gateway owns failover between providers, and an upstream 429 it passes
|
|
48
|
+
# on has already been decided there, so the SDK's own retries default to off
|
|
49
|
+
# rather than to openai-ruby's two. A nil timeout keeps openai-ruby's default.
|
|
45
50
|
def inference_client
|
|
51
|
+
config = NitroIntelligence.config
|
|
46
52
|
@inference_client ||= OpenAI::Client.new(
|
|
47
|
-
api_key:
|
|
48
|
-
base_url:
|
|
53
|
+
api_key: config.inference_api_key,
|
|
54
|
+
base_url: config.inference_base_url,
|
|
55
|
+
max_retries: @inference_max_retries.nil? ? config.inference_max_retries : @inference_max_retries,
|
|
56
|
+
**{ timeout: @inference_timeout || config.inference_timeout }.compact
|
|
49
57
|
)
|
|
50
58
|
end
|
|
51
59
|
end
|
|
@@ -40,7 +40,9 @@ module NitroIntelligence
|
|
|
40
40
|
},
|
|
41
41
|
},
|
|
42
42
|
}
|
|
43
|
-
|
|
43
|
+
# Deep, so that request options a caller passes (headers, max_retries,
|
|
44
|
+
# timeout) sit alongside the image config rather than replacing it.
|
|
45
|
+
parameters.replace(default_parameters.deep_merge(parameters))
|
|
44
46
|
Client.validate_model(parameters[:model])
|
|
45
47
|
end
|
|
46
48
|
|
|
@@ -10,6 +10,10 @@ module NitroIntelligence
|
|
|
10
10
|
|
|
11
11
|
MAX_STATUS_MESSAGE_LENGTH = 2000
|
|
12
12
|
|
|
13
|
+
# Attributes a handler may report off a response, each written only when it supplied
|
|
14
|
+
# one. See #record_result for why absence is left as absence.
|
|
15
|
+
RECORDED_ATTRIBUTES = %i[model model_parameters usage_details cost_details input output].freeze
|
|
16
|
+
|
|
13
17
|
attr_reader :project_client
|
|
14
18
|
|
|
15
19
|
def initialize(project_client:)
|
|
@@ -63,15 +67,30 @@ module NitroIntelligence
|
|
|
63
67
|
|
|
64
68
|
handle_truncation(trace_attributes[:input], trace_attributes[:output], trace_attributes[:model])
|
|
65
69
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
70
|
+
RECORDED_ATTRIBUTES.each do |name|
|
|
71
|
+
value = trace_attributes[name]
|
|
72
|
+
generation.public_send(:"#{name}=", value) if value
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
record_status(generation, trace_attributes)
|
|
71
76
|
|
|
72
77
|
generation.update_trace(input: trace_attributes[:input], output: trace_attributes[:output])
|
|
73
78
|
end
|
|
74
79
|
|
|
80
|
+
# A response the endpoint answered but did not finish -- a generation cut off at its
|
|
81
|
+
# token ceiling, say -- is neither a success worth nothing being said about nor a
|
|
82
|
+
# failure the request raised on. Recording it at WARNING leaves it findable without
|
|
83
|
+
# putting it among the errors.
|
|
84
|
+
def record_status(generation, trace_attributes)
|
|
85
|
+
status_message = trace_attributes[:status_message]
|
|
86
|
+
return if status_message.blank?
|
|
87
|
+
|
|
88
|
+
generation.update(
|
|
89
|
+
level: trace_attributes[:level] || "WARNING",
|
|
90
|
+
status_message: status_message.truncate(MAX_STATUS_MESSAGE_LENGTH)
|
|
91
|
+
)
|
|
92
|
+
end
|
|
93
|
+
|
|
75
94
|
# Recorded before the request is made so that a request which raises still
|
|
76
95
|
# shows what was sent. Handlers whose input is not safe to record twice
|
|
77
96
|
# (image generation sends base64 payloads that are replaced with media
|
|
@@ -13,6 +13,8 @@ module NitroIntelligence
|
|
|
13
13
|
config_accessor :assistants_config, default: {}
|
|
14
14
|
config_accessor :inference_api_key, default: ""
|
|
15
15
|
config_accessor :inference_base_url, default: "https://inference.powerhome.ai"
|
|
16
|
+
config_accessor :inference_max_retries, default: 0
|
|
17
|
+
config_accessor :inference_timeout, default: nil
|
|
16
18
|
config_accessor :model_config, default: {}
|
|
17
19
|
config_accessor :observability_base_url, default: "https://cerebro.powerhome.ai"
|
|
18
20
|
config_accessor :observability_projects, default: []
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
require "active_support/deprecation"
|
|
2
|
+
|
|
3
|
+
module NitroIntelligence
|
|
4
|
+
# Removed in the next major release. A Rails host has it registered with its own deprecators by
|
|
5
|
+
# the Railtie, so its `config.active_support` deprecation settings (log, raise, silence) apply;
|
|
6
|
+
# elsewhere it warns on stderr.
|
|
7
|
+
def self.deprecator
|
|
8
|
+
@deprecator ||= ActiveSupport::Deprecation.new("5.0", "nitro_intelligence")
|
|
9
|
+
end
|
|
10
|
+
end
|
|
@@ -4,6 +4,9 @@
|
|
|
4
4
|
# The content of this file should eventually make its way upstream. Setting a custom trace ID is
|
|
5
5
|
# already awaiting approval in https://github.com/simplepractice/langfuse-rb/pull/69.
|
|
6
6
|
#
|
|
7
|
+
# Explicit trace IDs create real roots here while the upstream fix awaits approval:
|
|
8
|
+
# https://github.com/simplepractice/langfuse-rb/pull/128.
|
|
9
|
+
#
|
|
7
10
|
|
|
8
11
|
require "nitro_intelligence/langfuse_tracer_provider"
|
|
9
12
|
|
|
@@ -50,11 +53,15 @@ module NitroIntelligence
|
|
|
50
53
|
Langfuse.send(:validate_observation_type!, as_type, type_str) unless skip_validation
|
|
51
54
|
|
|
52
55
|
otel_tracer = @tracer_provider.tracer
|
|
53
|
-
otel_span =
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
56
|
+
otel_span = if trace_id
|
|
57
|
+
@tracer_provider.start_root_span(name, trace_id: parent_span_context.trace_id, start_time:)
|
|
58
|
+
else
|
|
59
|
+
Langfuse.send(:create_otel_span,
|
|
60
|
+
name:,
|
|
61
|
+
start_time:,
|
|
62
|
+
parent_span_context:,
|
|
63
|
+
otel_tracer:)
|
|
64
|
+
end
|
|
58
65
|
|
|
59
66
|
# Serialize attributes
|
|
60
67
|
# Only set attributes if span is still recording (should always be true here, but guard for safety)
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
module NitroIntelligence
|
|
8
8
|
class LangfuseTracerProvider
|
|
9
9
|
def initialize(config)
|
|
10
|
+
@trace_id_key = OpenTelemetry::Context.create_key("nitro-intelligence-trace-id")
|
|
10
11
|
@tracer_provider = create_tracer_provider(config)
|
|
11
12
|
end
|
|
12
13
|
|
|
@@ -18,6 +19,25 @@ module NitroIntelligence
|
|
|
18
19
|
@tracer_provider.shutdown(timeout:)
|
|
19
20
|
end
|
|
20
21
|
|
|
22
|
+
def start_root_span(name, trace_id:, start_time: nil)
|
|
23
|
+
# A seed identifies the trace, not a missing parent. Preserve propagated
|
|
24
|
+
# attributes while clearing the ambient span for this explicit root.
|
|
25
|
+
context = OpenTelemetry::Trace.context_with_span(OpenTelemetry::Trace::Span::INVALID)
|
|
26
|
+
context = context.set_value(@trace_id_key, trace_id)
|
|
27
|
+
# generate_trace_id has no context argument, so expose the seed through Context.current.
|
|
28
|
+
OpenTelemetry::Context.with_current(context) do
|
|
29
|
+
tracer.start_span(name, with_parent: context, start_timestamp: start_time)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def generate_trace_id
|
|
34
|
+
OpenTelemetry::Context.value(@trace_id_key) || OpenTelemetry::Trace.generate_trace_id
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def generate_span_id
|
|
38
|
+
OpenTelemetry::Trace.generate_span_id
|
|
39
|
+
end
|
|
40
|
+
|
|
21
41
|
private
|
|
22
42
|
|
|
23
43
|
def create_tracer_provider(config)
|
|
@@ -36,7 +56,7 @@ module NitroIntelligence
|
|
|
36
56
|
max_export_batch_size: config.batch_size
|
|
37
57
|
)
|
|
38
58
|
|
|
39
|
-
tracer_provider = OpenTelemetry::SDK::Trace::TracerProvider.new
|
|
59
|
+
tracer_provider = OpenTelemetry::SDK::Trace::TracerProvider.new(id_generator: self)
|
|
40
60
|
tracer_provider.add_span_processor(processor)
|
|
41
61
|
span_processor = Langfuse::SpanProcessor.new(config:)
|
|
42
62
|
tracer_provider.add_span_processor(span_processor)
|
|
@@ -53,6 +73,7 @@ module NitroIntelligence
|
|
|
53
73
|
encoded = Base64.strict_encode64(credentials)
|
|
54
74
|
{
|
|
55
75
|
"Authorization" => "Basic #{encoded}",
|
|
76
|
+
"x-langfuse-ingestion-version" => "4",
|
|
56
77
|
}
|
|
57
78
|
end
|
|
58
79
|
end
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
require "active_support/core_ext/hash/indifferent_access"
|
|
2
|
+
require "nitro_intelligence/review_decision"
|
|
3
|
+
require "nitro_intelligence/tool_call_review_interrupt"
|
|
4
|
+
require "nitro_intelligence/tool_call_review_validator"
|
|
5
|
+
|
|
6
|
+
module NitroIntelligence
|
|
7
|
+
# What a paused run is waiting on: the interrupt the thread is holding, and the decisions that
|
|
8
|
+
# answer it.
|
|
9
|
+
#
|
|
10
|
+
# Assistants pauses for two kinds of review, never mixed in one interrupt:
|
|
11
|
+
#
|
|
12
|
+
# - tool calls, held by LangChain's `HumanInTheLoopMiddleware`, one action per call;
|
|
13
|
+
# - the run's final reply, a single action named `__reply__` whose `args` carry the draft's
|
|
14
|
+
# `content`.
|
|
15
|
+
#
|
|
16
|
+
# A turn with tool calls only ever pauses on its tool calls, and the reply is the run's final
|
|
17
|
+
# message, so `#tool_calls` and `#reply` are never both present.
|
|
18
|
+
#
|
|
19
|
+
# Built from a thread state (`GET /threads/{id}/state`, which reports `interrupts`) or from the
|
|
20
|
+
# response of a paused `/runs/wait` (which reports `__interrupt__`); both carry the messages the
|
|
21
|
+
# tool calls are matched back onto.
|
|
22
|
+
class PendingReview
|
|
23
|
+
REPLY_ACTION = "__reply__".freeze
|
|
24
|
+
|
|
25
|
+
# What a reviewer may do with a reply. LangChain's `respond` supplies a tool's result, which a
|
|
26
|
+
# reply does not have, so it is never offered for one, whatever the interrupt lists.
|
|
27
|
+
REPLY_DECISIONS = %w[approve edit reject].freeze
|
|
28
|
+
|
|
29
|
+
KIND_NAMES = { tool_calls: "tool calls", reply: "a reply" }.freeze
|
|
30
|
+
|
|
31
|
+
# The draft reply a reviewer is asked to approve, edit or reject.
|
|
32
|
+
Reply = Struct.new(:content, :allowed_decisions, :description, :interrupt_id, keyword_init: true)
|
|
33
|
+
|
|
34
|
+
# The pending review in a thread state, or nil when the thread is not paused.
|
|
35
|
+
def self.from_thread_state(thread_state)
|
|
36
|
+
interrupt = Array(thread_state["interrupts"]).first
|
|
37
|
+
interrupt && new(interrupt:, messages: Array(thread_state.dig("values", "messages")))
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# The pending review in a `/runs/wait` response, or nil when the run did not pause.
|
|
41
|
+
def self.from_run(run)
|
|
42
|
+
interrupt = Array(run["__interrupt__"]).first
|
|
43
|
+
interrupt && new(interrupt:, messages: Array(run["messages"]))
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def initialize(interrupt:, messages:)
|
|
47
|
+
@interrupt = interrupt.to_h
|
|
48
|
+
@messages = messages
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Identifies this pause. A thread that has moved on -- a new message superseded the review, or
|
|
52
|
+
# another reviewer decided it first -- reports a different interrupt, or none.
|
|
53
|
+
def interrupt_id
|
|
54
|
+
@interrupt["id"] || @interrupt["interrupt_id"]
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# `:tool_calls` or `:reply`.
|
|
58
|
+
def kind
|
|
59
|
+
action_name == REPLY_ACTION ? :reply : :tool_calls
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def tool_calls?
|
|
63
|
+
kind == :tool_calls
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def reply?
|
|
67
|
+
kind == :reply
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# The tool calls the interrupt is holding, as `Assistants#tool_calls_under_review` reports them.
|
|
71
|
+
# Empty when the pause is on a reply.
|
|
72
|
+
def tool_calls
|
|
73
|
+
return [] unless tool_calls?
|
|
74
|
+
|
|
75
|
+
tool_call_interrupt.tool_calls
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# The held draft, or nil when the pause is not on a reply.
|
|
79
|
+
def reply
|
|
80
|
+
return nil unless reply?
|
|
81
|
+
|
|
82
|
+
@reply ||= Reply.new(
|
|
83
|
+
content: action_args["content"],
|
|
84
|
+
allowed_decisions:,
|
|
85
|
+
description: action_requests.first["description"],
|
|
86
|
+
interrupt_id:
|
|
87
|
+
)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# The `command.resume` that sends `decisions` to this pause, keyed by its interrupt. LangGraph
|
|
91
|
+
# applies a keyed resume to that interrupt only: keyed by one that is no longer pending -- the
|
|
92
|
+
# thread moved on between reading this pause and resuming it -- it decides nothing, and the
|
|
93
|
+
# thread stays paused on the newer interrupt. An interrupt reported without an id can only be
|
|
94
|
+
# resumed unkeyed.
|
|
95
|
+
def resume(decisions)
|
|
96
|
+
interrupt_id.blank? ? { decisions: } : { interrupt_id => { decisions: } }
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Whether `run`, the response to a keyed resume of this pause, decided nothing because the
|
|
100
|
+
# interrupt was gone by the time the resume arrived. `run_id` is the run Assistants created for
|
|
101
|
+
# the resume, and the block reads the thread's state, only when it is needed.
|
|
102
|
+
#
|
|
103
|
+
# Such a resume comes back paused on the newer interrupt, exactly as a resume that was applied
|
|
104
|
+
# and paused again does -- a turn's tool calls decided, then the run's reply held. What tells
|
|
105
|
+
# them apart is who wrote the thread's latest checkpoint: Assistants records each run's id in
|
|
106
|
+
# the metadata of the checkpoints it writes, and a resume that decided nothing writes none, so
|
|
107
|
+
# the pause it reports still sits on a checkpoint an earlier run wrote.
|
|
108
|
+
#
|
|
109
|
+
# Best-effort, and only ever true on that evidence: without the run's id, the checkpoint's, or a
|
|
110
|
+
# state still on the reported pause (a new message may already have moved it on), or when the
|
|
111
|
+
# state cannot be read, it is false and the result stands as returned. That a stale decision is
|
|
112
|
+
# never applied is guaranteed by the keyed resume, not by this.
|
|
113
|
+
def unapplied_resume?(run:, run_id:)
|
|
114
|
+
paused_on = self.class.from_run(run)&.interrupt_id
|
|
115
|
+
return false if interrupt_id.blank? || run_id.blank? || paused_on.blank? || paused_on == interrupt_id
|
|
116
|
+
|
|
117
|
+
state = yield
|
|
118
|
+
writer = state.dig("metadata", "run_id")
|
|
119
|
+
self.class.from_thread_state(state)&.interrupt_id == paused_on && writer.present? && writer != run_id
|
|
120
|
+
rescue
|
|
121
|
+
false
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# The `command.resume` decisions for a review of this pause, validated against what the
|
|
125
|
+
# interrupt allows before anything is built, raising `Assistants::ThreadResumptionError`.
|
|
126
|
+
#
|
|
127
|
+
# For a reply, `reviews` is a single review hash. For tool calls, it is either a hash of reviews
|
|
128
|
+
# keyed by tool call id, as `Assistants#review_tool_calls` has always taken them, or a list of
|
|
129
|
+
# reviews that each carry their tool call's `id`.
|
|
130
|
+
def decisions(reviews)
|
|
131
|
+
return [reply_decision(reviews)] if reply?
|
|
132
|
+
|
|
133
|
+
reviews = tool_call_reviews(reviews)
|
|
134
|
+
ToolCallReviewValidator.new.validate!(tool_calls: reviews, tool_calls_under_review: tool_calls)
|
|
135
|
+
tool_call_interrupt.decisions(reviews)
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
private
|
|
139
|
+
|
|
140
|
+
def tool_call_interrupt
|
|
141
|
+
@tool_call_interrupt ||= ToolCallReviewInterrupt.new(
|
|
142
|
+
"interrupts" => [@interrupt],
|
|
143
|
+
"values" => { "messages" => @messages }
|
|
144
|
+
)
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def interrupt_value
|
|
148
|
+
@interrupt["value"] || {}
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def action_requests
|
|
152
|
+
Array(interrupt_value["action_requests"])
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def action_name
|
|
156
|
+
action_requests.first&.dig("name")
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def action_args
|
|
160
|
+
action_requests.first["args"] || {}
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def allowed_decisions
|
|
164
|
+
review_config = Array(interrupt_value["review_configs"]).find do |config|
|
|
165
|
+
config["action_name"] == action_name
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
Array(review_config&.dig("allowed_decisions")).map(&:to_s) & REPLY_DECISIONS
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def tool_call_reviews(reviews)
|
|
172
|
+
return reviews unless reviews.is_a?(Array)
|
|
173
|
+
|
|
174
|
+
reviews.each_with_object({}) do |review, by_id|
|
|
175
|
+
unless review.is_a?(Hash) && review.with_indifferent_access[:id].present?
|
|
176
|
+
raise_error!("Each tool call review in a list must be a hash with the tool call's `id`")
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
review = review.with_indifferent_access
|
|
180
|
+
id = review[:id].to_s
|
|
181
|
+
raise_error!("Tool call `#{id}` is reviewed more than once") if by_id.key?(id)
|
|
182
|
+
|
|
183
|
+
by_id[id] = review.except(:id)
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# A reply is a single action, so it is reviewed with a single hash.
|
|
188
|
+
def reply_decision(review)
|
|
189
|
+
raise_error!("The reply review must be a hash") unless review.is_a?(Hash)
|
|
190
|
+
|
|
191
|
+
review = review.with_indifferent_access
|
|
192
|
+
action = review[:action].to_s
|
|
193
|
+
|
|
194
|
+
unless allowed_decisions.include?(action)
|
|
195
|
+
raise_error!("Invalid review action `#{action}` for the reply; allowed: #{allowed_decisions.join(', ')}")
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
case action
|
|
199
|
+
when "approve" then ReviewDecision.approve
|
|
200
|
+
when "edit"
|
|
201
|
+
ReviewDecision.edit(REPLY_ACTION, { "content" => text!(review, :content, action:, required: true) })
|
|
202
|
+
when "reject" then ReviewDecision.reject(text!(review, :message, action:, required: false))
|
|
203
|
+
end
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# A required field must be non-blank text; an optional one may be absent, but not something other
|
|
207
|
+
# than a String.
|
|
208
|
+
def text!(review, field, action:, required:)
|
|
209
|
+
value = review[field]
|
|
210
|
+
case ReviewDecision.message_problem(value, required:)
|
|
211
|
+
when :missing then raise_error!("A `#{action}` review of the reply must include a non-empty `#{field}`")
|
|
212
|
+
when :not_string then raise_error!("`#{field}` for the reply review must be a string")
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
value
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def raise_error!(message)
|
|
219
|
+
raise Assistants::ThreadResumptionError, message
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
end
|
|
@@ -35,12 +35,13 @@ module NitroIntelligence
|
|
|
35
35
|
response
|
|
36
36
|
end
|
|
37
37
|
|
|
38
|
-
def score(trace_id:, name:, value:, id: "#{trace_id}-#{name}")
|
|
38
|
+
def score(trace_id:, name:, value:, id: "#{trace_id}-#{name}", data_type: :numeric)
|
|
39
39
|
@project_client.observability_client.create_score(
|
|
40
40
|
id:,
|
|
41
41
|
trace_id:,
|
|
42
42
|
name:,
|
|
43
43
|
value:,
|
|
44
|
+
data_type:,
|
|
44
45
|
environment: NitroIntelligence.environment
|
|
45
46
|
)
|
|
46
47
|
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
module NitroIntelligence
|
|
2
|
+
# The `command.resume` decisions a review sends, and the check on the `message` a decision
|
|
3
|
+
# carries, shared by tool call reviews and reply reviews. Each caller keeps its own
|
|
4
|
+
# error wording, so this reports what is wrong rather than raising.
|
|
5
|
+
module ReviewDecision
|
|
6
|
+
module_function
|
|
7
|
+
|
|
8
|
+
def approve
|
|
9
|
+
{ "type" => "approve" }
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def edit(name, args)
|
|
13
|
+
{ "type" => "edit", "edited_action" => { "name" => name, "args" => args } }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# The middleware falls back to its own wording when a rejection carries no reason.
|
|
17
|
+
def reject(message)
|
|
18
|
+
message.nil? ? { "type" => "reject" } : { "type" => "reject", "message" => message }
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def respond(message)
|
|
22
|
+
{ "type" => "respond", "message" => message }
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# What is wrong with a decision's `message`, or nil: `:missing` when a required one is absent
|
|
26
|
+
# or blank, `:not_string` when one is given as anything but a String.
|
|
27
|
+
def message_problem(message, required:)
|
|
28
|
+
return (:missing if required) if message.nil?
|
|
29
|
+
return :not_string unless message.is_a?(String)
|
|
30
|
+
|
|
31
|
+
:missing if required && message.blank?
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
require "nitro_intelligence/pending_review"
|
|
2
|
+
|
|
3
|
+
module NitroIntelligence
|
|
4
|
+
# The outcome of one `/runs/wait`: either the run ended, possibly with a reply, or it paused on a
|
|
5
|
+
# review.
|
|
6
|
+
#
|
|
7
|
+
# A run has at most one reply: its final message, an AI message with text and no tool calls. Text
|
|
8
|
+
# the model writes alongside tool calls is working text, never the reply. A paused run's last AI
|
|
9
|
+
# message is a draft awaiting review, so `#reply` is nil whenever the run paused; a reply is final
|
|
10
|
+
# only once a run ends without `__interrupt__`.
|
|
11
|
+
class RunResult
|
|
12
|
+
# How the platform delivered the reply through the assistant's delivery tool, read from the
|
|
13
|
+
# reply's `response_metadata.delivery` stamp. `status` is "delivered" (with `result`, what the
|
|
14
|
+
# tool returned, and `delivered_at`), "failed" (with the tool's `error` and `failed_at`), or
|
|
15
|
+
# "pending": the platform stamps a settled reply "pending" just before it delivers it, so a
|
|
16
|
+
# finished run still showing it was cut off before delivery recorded an outcome. Treat a
|
|
17
|
+
# pending reply as not sent; the next customer message removes it and the model answers both. A
|
|
18
|
+
# field the stamp does not carry, or carries as the wrong type, is nil.
|
|
19
|
+
Delivery = Struct.new(:status, :tool, :attempts, :error, :result, :delivered_at, :failed_at,
|
|
20
|
+
keyword_init: true) do
|
|
21
|
+
def delivered?
|
|
22
|
+
status == "delivered"
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def failed?
|
|
26
|
+
status == "failed"
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def pending?
|
|
30
|
+
status == "pending"
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# The `response_metadata.review` type the platform stamps on a draft when it pauses on it.
|
|
35
|
+
PENDING_REVIEW = "pending".freeze
|
|
36
|
+
|
|
37
|
+
# The response exactly as Assistants returned it.
|
|
38
|
+
attr_reader :raw
|
|
39
|
+
|
|
40
|
+
def initialize(run)
|
|
41
|
+
@raw = run.to_h
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def interrupted?
|
|
45
|
+
Array(raw["__interrupt__"]).any?
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# What the paused run is waiting on, or nil when it did not pause.
|
|
49
|
+
def review
|
|
50
|
+
return @review if defined?(@review)
|
|
51
|
+
|
|
52
|
+
@review = PendingReview.from_run(raw)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# The run's messages as Assistants reports them, oldest first.
|
|
56
|
+
def messages
|
|
57
|
+
Array(raw["messages"])
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# The text of the run's reply, always a String: content given as blocks is reduced to its text.
|
|
61
|
+
# Nil while the run is paused, and when it ended without one: a rejected draft is removed,
|
|
62
|
+
# leaving a tool result or the human's message last, and an earlier turn's text is never the
|
|
63
|
+
# reply. Blank text is no reply.
|
|
64
|
+
def reply
|
|
65
|
+
reply_message && text(reply_message["content"]).presence
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# The reply's delivery, or nil when the reply carries no delivery stamp: no reply, or no
|
|
69
|
+
# delivery tool configured for the assistant.
|
|
70
|
+
def delivery
|
|
71
|
+
return @delivery if defined?(@delivery)
|
|
72
|
+
|
|
73
|
+
stamp = reply_message&.dig("response_metadata", "delivery")
|
|
74
|
+
@delivery = stamp.is_a?(Hash) ? build_delivery(stamp) : nil
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
private
|
|
78
|
+
|
|
79
|
+
# The run's last message, when the run ended and that message is the assistant's reply: an AI
|
|
80
|
+
# message with no tool calls. A draft still stamped as awaiting review is never the reply, even
|
|
81
|
+
# in a response that reports no pause.
|
|
82
|
+
def reply_message
|
|
83
|
+
return nil if interrupted?
|
|
84
|
+
|
|
85
|
+
message = messages.last
|
|
86
|
+
return nil unless message.is_a?(Hash)
|
|
87
|
+
return nil if message["type"].present? && message["type"] != "ai"
|
|
88
|
+
return nil if Array(message["tool_calls"]).any?
|
|
89
|
+
return nil if awaiting_review?(message)
|
|
90
|
+
|
|
91
|
+
message
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def awaiting_review?(message)
|
|
95
|
+
review = message.dig("response_metadata", "review")
|
|
96
|
+
review.is_a?(Hash) && review["type"] == PENDING_REVIEW
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# A message's content is a string, or a list of content blocks. Its text is the string, or the
|
|
100
|
+
# bare strings and `text` blocks joined, as the platform reads it (LangChain's `message.text`);
|
|
101
|
+
# other blocks -- tool use, reasoning, images -- are not part of the reply.
|
|
102
|
+
def text(content)
|
|
103
|
+
return content if content.is_a?(String)
|
|
104
|
+
return nil unless content.is_a?(Array)
|
|
105
|
+
|
|
106
|
+
content.filter_map do |block|
|
|
107
|
+
next block if block.is_a?(String)
|
|
108
|
+
|
|
109
|
+
block["text"] if block.is_a?(Hash) && block["type"] == "text" && block["text"].is_a?(String)
|
|
110
|
+
end.join
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def build_delivery(stamp)
|
|
114
|
+
Delivery.new(
|
|
115
|
+
status: string_or_nil(stamp["status"]),
|
|
116
|
+
tool: string_or_nil(stamp["tool"]),
|
|
117
|
+
attempts: stamp["attempts"].is_a?(Integer) ? stamp["attempts"] : nil,
|
|
118
|
+
error: string_or_nil(stamp["error"]),
|
|
119
|
+
result: stamp["result"],
|
|
120
|
+
delivered_at: string_or_nil(stamp["delivered_at"]),
|
|
121
|
+
failed_at: string_or_nil(stamp["failed_at"])
|
|
122
|
+
)
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def string_or_nil(value)
|
|
126
|
+
value.is_a?(String) ? value : nil
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
end
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require "active_support/core_ext/hash/indifferent_access"
|
|
2
|
+
require "nitro_intelligence/review_decision"
|
|
2
3
|
|
|
3
4
|
module NitroIntelligence
|
|
4
5
|
# The tool calls one interrupt is holding, and the resume payload that answers them.
|
|
@@ -68,14 +69,10 @@ module NitroIntelligence
|
|
|
68
69
|
review = (review || {}).with_indifferent_access
|
|
69
70
|
|
|
70
71
|
case review[:action].to_s
|
|
71
|
-
when "approve"
|
|
72
|
-
|
|
73
|
-
when "
|
|
74
|
-
|
|
75
|
-
when "reject"
|
|
76
|
-
review[:message].nil? ? { "type" => "reject" } : { "type" => "reject", "message" => review[:message] }
|
|
77
|
-
when "respond"
|
|
78
|
-
{ "type" => "respond", "message" => review[:message] }
|
|
72
|
+
when "approve" then ReviewDecision.approve
|
|
73
|
+
when "edit" then ReviewDecision.edit(tool_call["name"], edited_args(tool_call, review))
|
|
74
|
+
when "reject" then ReviewDecision.reject(review[:message])
|
|
75
|
+
when "respond" then ReviewDecision.respond(review[:message])
|
|
79
76
|
else
|
|
80
77
|
raise Assistants::ThreadResumptionError,
|
|
81
78
|
"Tool call #{tool_call['id']} has no review naming `approve`, `edit`, `reject` or `respond`"
|
|
@@ -84,11 +81,8 @@ module NitroIntelligence
|
|
|
84
81
|
|
|
85
82
|
# Edited arguments are merged over the call the model made, so a reviewer changing one of them
|
|
86
83
|
# does not have to restate the rest -- and cannot drop one by omitting it.
|
|
87
|
-
def
|
|
88
|
-
{
|
|
89
|
-
"name" => tool_call["name"],
|
|
90
|
-
"args" => tool_call["args"].merge(review[:args] || {}),
|
|
91
|
-
}
|
|
84
|
+
def edited_args(tool_call, review)
|
|
85
|
+
tool_call["args"].merge(review[:args] || {})
|
|
92
86
|
end
|
|
93
87
|
|
|
94
88
|
def interrupt_value
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require "active_support/core_ext/hash/indifferent_access"
|
|
2
|
+
require "nitro_intelligence/review_decision"
|
|
2
3
|
|
|
3
4
|
module NitroIntelligence
|
|
4
5
|
# The reviews a caller submitted, checked against the tool calls the interrupt is holding, before
|
|
@@ -72,15 +73,10 @@ module NitroIntelligence
|
|
|
72
73
|
end
|
|
73
74
|
|
|
74
75
|
def validate_message!(tool_call_id:, review:, required:)
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
raise_error!("Review for tool call #{tool_call_id} must include a message") if required
|
|
79
|
-
return
|
|
76
|
+
case ReviewDecision.message_problem(review[:message], required:)
|
|
77
|
+
when :missing then raise_error!("Review for tool call #{tool_call_id} must include a message")
|
|
78
|
+
when :not_string then raise_error!("Message for tool call #{tool_call_id} must be a string")
|
|
80
79
|
end
|
|
81
|
-
|
|
82
|
-
raise_error!("Message for tool call #{tool_call_id} must be a string") unless message.is_a?(String)
|
|
83
|
-
raise_error!("Review for tool call #{tool_call_id} must include a message") if required && message.blank?
|
|
84
80
|
end
|
|
85
81
|
|
|
86
82
|
def validate_completeness!(submitted_tool_call_ids:, tool_calls_under_review:)
|