riffer 0.47.2 → 0.48.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.release-please-manifest.json +1 -1
- data/CHANGELOG.md +23 -0
- data/docs/AGENTS.md +37 -3
- data/docs/AGENT_LIFECYCLE.md +1 -0
- data/docs/CONFIGURATION.md +1 -1
- data/docs/EVALS.md +2 -1
- data/docs/MCP.md +0 -4
- data/docs/MESSAGES.md +85 -17
- data/docs/STREAM_EVENTS.md +8 -4
- data/docs/TRACING.md +1 -1
- data/docs/providers/AMAZON_BEDROCK.md +1 -1
- data/docs/providers/CUSTOM_PROVIDERS.md +56 -4
- data/docs/providers/GEMINI.md +1 -1
- data/docs/providers/MOCK_PROVIDER.md +17 -0
- data/lib/riffer/agent/config.rb +30 -6
- data/lib/riffer/agent/response.rb +7 -0
- data/lib/riffer/agent/run.rb +24 -8
- data/lib/riffer/agent/session.rb +1 -0
- data/lib/riffer/agent.rb +29 -5
- data/lib/riffer/config.rb +1 -1
- data/lib/riffer/evals/evaluator.rb +29 -2
- data/lib/riffer/evals/judge.rb +11 -5
- data/lib/riffer/files/resolver.rb +3 -3
- data/lib/riffer/helpers/deep_dup.rb +42 -0
- data/lib/riffer/messages/assistant/reasoning_part.rb +90 -0
- data/lib/riffer/messages/assistant/tool_call.rb +63 -0
- data/lib/riffer/messages/assistant.rb +50 -2
- data/lib/riffer/messages/base.rb +5 -21
- data/lib/riffer/messages/system.rb +10 -0
- data/lib/riffer/messages/tool.rb +17 -0
- data/lib/riffer/messages/{file_part.rb → user/file_part.rb} +15 -11
- data/lib/riffer/messages/user.rb +13 -2
- data/lib/riffer/params/param.rb +12 -0
- data/lib/riffer/params.rb +9 -0
- data/lib/riffer/providers/amazon_bedrock.rb +5 -4
- data/lib/riffer/providers/anthropic.rb +2 -2
- data/lib/riffer/providers/base.rb +27 -8
- data/lib/riffer/providers/gemini.rb +2 -2
- data/lib/riffer/providers/mock.rb +18 -2
- data/lib/riffer/providers/open_ai.rb +6 -5
- data/lib/riffer/providers/open_router.rb +6 -5
- data/lib/riffer/providers/token_usage.rb +16 -0
- data/lib/riffer/skills/config.rb +12 -0
- data/lib/riffer/stream_events/reasoning_done.rb +10 -8
- data/lib/riffer/tools/runtime.rb +3 -4
- data/lib/riffer/tracing/capture.rb +2 -4
- data/lib/riffer/version.rb +1 -1
- data/sig/generated/riffer/agent/config.rbs +19 -5
- data/sig/generated/riffer/agent/response.rbs +6 -1
- data/sig/generated/riffer/agent/run.rbs +15 -8
- data/sig/generated/riffer/agent.rbs +25 -9
- data/sig/generated/riffer/config.rbs +1 -1
- data/sig/generated/riffer/evals/evaluator.rbs +20 -1
- data/sig/generated/riffer/evals/judge.rbs +5 -2
- data/sig/generated/riffer/files/resolver.rbs +6 -6
- data/sig/generated/riffer/helpers/deep_dup.rbs +21 -0
- data/sig/generated/riffer/messages/assistant/reasoning_part.rbs +58 -0
- data/sig/generated/riffer/messages/assistant/tool_call.rbs +42 -0
- data/sig/generated/riffer/messages/assistant.rbs +24 -10
- data/sig/generated/riffer/messages/system.rbs +6 -0
- data/sig/generated/riffer/messages/tool.rbs +6 -0
- data/sig/generated/riffer/messages/{file_part.rbs → user/file_part.rbs} +11 -6
- data/sig/generated/riffer/messages/user.rbs +9 -3
- data/sig/generated/riffer/params/param.rbs +7 -0
- data/sig/generated/riffer/params.rbs +6 -0
- data/sig/generated/riffer/providers/amazon_bedrock.rbs +4 -4
- data/sig/generated/riffer/providers/anthropic.rbs +2 -2
- data/sig/generated/riffer/providers/base.rbs +22 -10
- data/sig/generated/riffer/providers/gemini.rbs +4 -4
- data/sig/generated/riffer/providers/mock.rbs +7 -2
- data/sig/generated/riffer/providers/open_ai.rbs +4 -4
- data/sig/generated/riffer/providers/open_router.rbs +4 -4
- data/sig/generated/riffer/providers/token_usage.rbs +6 -0
- data/sig/generated/riffer/skills/config.rbs +9 -0
- data/sig/generated/riffer/stream_events/reasoning_done.rbs +8 -6
- data/sig/generated/riffer/tools/runtime.rbs +2 -2
- data/sig/generated/riffer/tracing/capture.rbs +4 -4
- data/sig/manual/riffer/helpers/deep_dup.rbs +5 -0
- metadata +10 -3
data/lib/riffer/agent/run.rb
CHANGED
|
@@ -10,7 +10,7 @@ module Riffer::Agent::Run
|
|
|
10
10
|
# for prompt/files semantics.
|
|
11
11
|
#
|
|
12
12
|
#--
|
|
13
|
-
#: (agent: Riffer::Agent, ?prompt: String?, ?files: Array[Hash[Symbol, untyped] | Riffer::Messages::FilePart]?, ?tags: Hash[(String | Symbol), untyped]) -> Riffer::Agent::Response
|
|
13
|
+
#: (agent: Riffer::Agent, ?prompt: String?, ?files: Array[Hash[Symbol, untyped] | Riffer::Messages::User::FilePart]?, ?tags: Hash[(String | Symbol), untyped]) -> Riffer::Agent::Response
|
|
14
14
|
def generate(agent:, prompt: nil, files: nil, tags: {})
|
|
15
15
|
append_user_message(agent, prompt, files: files)
|
|
16
16
|
run_loop(agent, tags: tags)
|
|
@@ -20,7 +20,7 @@ module Riffer::Agent::Run
|
|
|
20
20
|
# for prompt/files semantics.
|
|
21
21
|
#
|
|
22
22
|
#--
|
|
23
|
-
#: (agent: Riffer::Agent, ?prompt: String?, ?files: Array[Hash[Symbol, untyped] | Riffer::Messages::FilePart]?, ?tags: Hash[(String | Symbol), untyped]) -> Enumerator[Riffer::StreamEvents::Base, Riffer::Agent::Response]
|
|
23
|
+
#: (agent: Riffer::Agent, ?prompt: String?, ?files: Array[Hash[Symbol, untyped] | Riffer::Messages::User::FilePart]?, ?tags: Hash[(String | Symbol), untyped]) -> Enumerator[Riffer::StreamEvents::Base, Riffer::Agent::Response]
|
|
24
24
|
def stream(agent:, prompt: nil, files: nil, tags: {})
|
|
25
25
|
append_user_message(agent, prompt, files: files)
|
|
26
26
|
# The enumerator body runs in its own fiber, where the fiber-local OTEL
|
|
@@ -36,16 +36,17 @@ module Riffer::Agent::Run
|
|
|
36
36
|
private
|
|
37
37
|
|
|
38
38
|
# Both +generate+ and +stream+ funnel here, so this is the single place raw
|
|
39
|
-
# +tags+ are normalized
|
|
39
|
+
# +tags+ are normalized and merged over the default tags (a caller tag wins
|
|
40
|
+
# on a shared key). The clean <tt>String => String</tt> map is then
|
|
40
41
|
# threaded to every span builder in the run as +riffer.tag.*+ and to each
|
|
41
42
|
# provider call (via +merged_model_options+) for native request-metadata
|
|
42
43
|
# mapping.
|
|
43
44
|
#--
|
|
44
45
|
#: (Riffer::Agent, ?tags: Hash[(String | Symbol), untyped]?, ?stream_yielder: Enumerator::Yielder?) -> Riffer::Agent::Response
|
|
45
46
|
def run_loop(agent, tags: {}, stream_yielder: nil)
|
|
46
|
-
tags = normalize_tags(tags)
|
|
47
|
+
tags = default_tags(agent).merge(normalize_tags(tags))
|
|
47
48
|
Riffer::Tracing.in_span(
|
|
48
|
-
"invoke_agent #{agent.
|
|
49
|
+
"invoke_agent #{agent.identifier}",
|
|
49
50
|
attributes: run_span_attributes(agent, tags),
|
|
50
51
|
kind: :internal,
|
|
51
52
|
) do |span|
|
|
@@ -153,6 +154,7 @@ module Riffer::Agent::Run
|
|
|
153
154
|
def accumulate_streamed_response(agent, stream_yielder, tags = {})
|
|
154
155
|
accumulated_content = +""
|
|
155
156
|
accumulated_tool_calls = [] #: Array[Riffer::Messages::Assistant::ToolCall]
|
|
157
|
+
accumulated_reasoning = [] #: Array[Riffer::Messages::Assistant::ReasoningPart]
|
|
156
158
|
accumulated_token_usage = nil #: Riffer::Providers::TokenUsage?
|
|
157
159
|
accumulated_finish_reason = nil #: Symbol?
|
|
158
160
|
accumulated_finish_reason_raw = nil #: String?
|
|
@@ -175,6 +177,8 @@ module Riffer::Agent::Run
|
|
|
175
177
|
name: event.name,
|
|
176
178
|
arguments: event.arguments,
|
|
177
179
|
)
|
|
180
|
+
when Riffer::StreamEvents::ReasoningDone
|
|
181
|
+
accumulated_reasoning << event.part
|
|
178
182
|
when Riffer::StreamEvents::TokenUsageDone
|
|
179
183
|
accumulated_token_usage = event.token_usage
|
|
180
184
|
when Riffer::StreamEvents::FinishReasonDone
|
|
@@ -186,6 +190,7 @@ module Riffer::Agent::Run
|
|
|
186
190
|
Riffer::Messages::Assistant.new(
|
|
187
191
|
accumulated_content,
|
|
188
192
|
tool_calls: accumulated_tool_calls,
|
|
193
|
+
reasoning: accumulated_reasoning,
|
|
189
194
|
token_usage: accumulated_token_usage,
|
|
190
195
|
finish_reason: accumulated_finish_reason,
|
|
191
196
|
finish_reason_raw: accumulated_finish_reason_raw,
|
|
@@ -228,6 +233,7 @@ module Riffer::Agent::Run
|
|
|
228
233
|
message&.content || "",
|
|
229
234
|
outcome: final_outcome(message, result, interrupted: interrupted, interrupt_reason: interrupt_reason),
|
|
230
235
|
modifications: all_modifications,
|
|
236
|
+
reasoning: message&.reasoning || [],
|
|
231
237
|
structured_output: result&.object,
|
|
232
238
|
**extra,
|
|
233
239
|
)
|
|
@@ -381,6 +387,13 @@ module Riffer::Agent::Run
|
|
|
381
387
|
opts
|
|
382
388
|
end
|
|
383
389
|
|
|
390
|
+
# The tags riffer adds to every run, identifying the agent it's on behalf of.
|
|
391
|
+
#--
|
|
392
|
+
#: (Riffer::Agent) -> Hash[String, String]
|
|
393
|
+
def default_tags(agent)
|
|
394
|
+
{ "kind" => "agent", "agent" => agent.identifier }
|
|
395
|
+
end
|
|
396
|
+
|
|
384
397
|
#--
|
|
385
398
|
#: (
|
|
386
399
|
# Riffer::Agent,
|
|
@@ -388,6 +401,7 @@ module Riffer::Agent::Run
|
|
|
388
401
|
# outcome: Riffer::Agent::Outcome,
|
|
389
402
|
# ?tripwire: Riffer::Guardrails::Tripwire?,
|
|
390
403
|
# ?modifications: Array[Riffer::Guardrails::Modification],
|
|
404
|
+
# ?reasoning: Array[Riffer::Messages::Assistant::ReasoningPart],
|
|
391
405
|
# ?structured_output: Hash[Symbol, untyped]?,
|
|
392
406
|
# ?healed_tool_call_ids: Array[String],
|
|
393
407
|
# ?token_usage: Riffer::Providers::TokenUsage?,
|
|
@@ -399,6 +413,7 @@ module Riffer::Agent::Run
|
|
|
399
413
|
outcome:,
|
|
400
414
|
tripwire: nil,
|
|
401
415
|
modifications: [],
|
|
416
|
+
reasoning: [],
|
|
402
417
|
structured_output: nil,
|
|
403
418
|
healed_tool_call_ids: [],
|
|
404
419
|
token_usage: nil,
|
|
@@ -410,6 +425,7 @@ module Riffer::Agent::Run
|
|
|
410
425
|
outcome: outcome,
|
|
411
426
|
tripwire: tripwire,
|
|
412
427
|
modifications: modifications,
|
|
428
|
+
reasoning: reasoning,
|
|
413
429
|
structured_output: structured_output,
|
|
414
430
|
messages: messages.frozen? ? messages : messages.dup.freeze,
|
|
415
431
|
healed_tool_call_ids: healed_tool_call_ids,
|
|
@@ -421,12 +437,12 @@ module Riffer::Agent::Run
|
|
|
421
437
|
# Raises when +files+ are supplied without a +prompt+ — the provider needs
|
|
422
438
|
# text to anchor the attachments.
|
|
423
439
|
#--
|
|
424
|
-
#: (Riffer::Agent, String?, ?files: Array[Hash[Symbol, untyped] | Riffer::Messages::FilePart]?) -> void
|
|
440
|
+
#: (Riffer::Agent, String?, ?files: Array[Hash[Symbol, untyped] | Riffer::Messages::User::FilePart]?) -> void
|
|
425
441
|
def append_user_message(agent, prompt, files: nil)
|
|
426
442
|
raise Riffer::ArgumentError, "files: requires a prompt" if files && !files.empty? && prompt.nil?
|
|
427
443
|
return unless prompt
|
|
428
444
|
|
|
429
|
-
file_parts = (files || []).map { |f| Riffer::Messages::FilePart.from_hash(f) }
|
|
445
|
+
file_parts = (files || []).map { |f| Riffer::Messages::User::FilePart.from_hash(f) }
|
|
430
446
|
agent.session.add(Riffer::Messages::User.new(prompt, files: file_parts), silent: true)
|
|
431
447
|
end
|
|
432
448
|
|
|
@@ -451,7 +467,7 @@ module Riffer::Agent::Run
|
|
|
451
467
|
def run_span_attributes(agent, tags = {})
|
|
452
468
|
{
|
|
453
469
|
"gen_ai.operation.name" => "invoke_agent",
|
|
454
|
-
"gen_ai.agent.name" => agent.
|
|
470
|
+
"gen_ai.agent.name" => agent.identifier,
|
|
455
471
|
"gen_ai.provider.name" => agent.provider.class.semconv_provider_name,
|
|
456
472
|
"gen_ai.request.model" => agent.model_name,
|
|
457
473
|
}.merge(tag_attributes(tags))
|
data/lib/riffer/agent/session.rb
CHANGED
|
@@ -201,6 +201,7 @@ class Riffer::Agent::Session
|
|
|
201
201
|
attrs.fetch(:content, old.content),
|
|
202
202
|
id: attrs.fetch(:id, old.id),
|
|
203
203
|
tool_calls: attrs.fetch(:tool_calls, old.tool_calls),
|
|
204
|
+
reasoning: attrs.fetch(:reasoning, old.reasoning),
|
|
204
205
|
token_usage: attrs.fetch(:token_usage, old.token_usage),
|
|
205
206
|
structured_output: attrs.fetch(:structured_output, old.structured_output),
|
|
206
207
|
finish_reason: attrs.fetch(:finish_reason, old.finish_reason),
|
data/lib/riffer/agent.rb
CHANGED
|
@@ -28,6 +28,19 @@ class Riffer::Agent
|
|
|
28
28
|
@config ||= Riffer::Agent::Config.new
|
|
29
29
|
end
|
|
30
30
|
|
|
31
|
+
# +identifier+ is cleared because it is configuration here but identity on the
|
|
32
|
+
# subclass; two classes claiming one raise Riffer::DuplicateIdentifierError at
|
|
33
|
+
# the next registry lookup.
|
|
34
|
+
#--
|
|
35
|
+
#: (Class) -> void
|
|
36
|
+
def self.inherited(subclass)
|
|
37
|
+
super
|
|
38
|
+
copy = config.dup
|
|
39
|
+
copy.identifier = nil
|
|
40
|
+
subclass.instance_variable_set(:@config, copy)
|
|
41
|
+
end
|
|
42
|
+
private_class_method :inherited
|
|
43
|
+
|
|
31
44
|
# Gets or sets the agent identifier.
|
|
32
45
|
#
|
|
33
46
|
#--
|
|
@@ -148,14 +161,14 @@ class Riffer::Agent
|
|
|
148
161
|
|
|
149
162
|
# Generates a response using a new agent instance.
|
|
150
163
|
#--
|
|
151
|
-
#: (?String?, ?files: Array[Hash[Symbol, untyped] | Riffer::Messages::FilePart]?, ?context: Hash[Symbol, untyped]?, ?tags: Hash[(String | Symbol), untyped]) -> Riffer::Agent::Response
|
|
164
|
+
#: (?String?, ?files: Array[Hash[Symbol, untyped] | Riffer::Messages::User::FilePart]?, ?context: Hash[Symbol, untyped]?, ?tags: Hash[(String | Symbol), untyped]) -> Riffer::Agent::Response
|
|
152
165
|
def self.generate(prompt = nil, files: nil, context: nil, tags: {})
|
|
153
166
|
new(context: context).generate(prompt, files: files, tags: tags)
|
|
154
167
|
end
|
|
155
168
|
|
|
156
169
|
# Streams a response using a new agent instance.
|
|
157
170
|
#--
|
|
158
|
-
#: (?String?, ?files: Array[Hash[Symbol, untyped] | Riffer::Messages::FilePart]?, ?context: Hash[Symbol, untyped]?, ?tags: Hash[(String | Symbol), untyped]) -> Enumerator[Riffer::StreamEvents::Base, Riffer::Agent::Response]
|
|
171
|
+
#: (?String?, ?files: Array[Hash[Symbol, untyped] | Riffer::Messages::User::FilePart]?, ?context: Hash[Symbol, untyped]?, ?tags: Hash[(String | Symbol), untyped]) -> Enumerator[Riffer::StreamEvents::Base, Riffer::Agent::Response]
|
|
159
172
|
def self.stream(prompt = nil, files: nil, context: nil, tags: {})
|
|
160
173
|
new(context: context).stream(prompt, files: files, tags: tags)
|
|
161
174
|
end
|
|
@@ -284,6 +297,16 @@ class Riffer::Agent
|
|
|
284
297
|
@session.set(Riffer::Agent::Session::Repair.prune_orphans(@session.messages))
|
|
285
298
|
end
|
|
286
299
|
|
|
300
|
+
# The identifier for this agent: the instance config's identifier when set
|
|
301
|
+
# (e.g. an agent restored with +Riffer::Agent.from_h+), otherwise the class
|
|
302
|
+
# identifier.
|
|
303
|
+
#
|
|
304
|
+
#--
|
|
305
|
+
#: () -> String
|
|
306
|
+
def identifier
|
|
307
|
+
config.identifier || self.class.identifier
|
|
308
|
+
end
|
|
309
|
+
|
|
287
310
|
# Generates a response from the agent.
|
|
288
311
|
#
|
|
289
312
|
# With +prompt+, a new user message is appended (silently — +on_message+ does
|
|
@@ -296,10 +319,11 @@ class Riffer::Agent
|
|
|
296
319
|
# are stamped as +riffer.tag.*+ on every span the call emits. See
|
|
297
320
|
# +docs/AGENTS.md+ for the per-provider mapping. The reserved key
|
|
298
321
|
# +user_id+ also maps to the provider's native user identifier where one
|
|
299
|
-
# exists.
|
|
322
|
+
# exists. Riffer adds the default tags +kind+ (+"agent"+) and +agent+ (the
|
|
323
|
+
# agent identifier); a caller tag with the same key wins.
|
|
300
324
|
#
|
|
301
325
|
#--
|
|
302
|
-
#: (?String?, ?files: Array[Hash[Symbol, untyped] | Riffer::Messages::FilePart]?, ?tags: Hash[(String | Symbol), untyped]) -> Riffer::Agent::Response
|
|
326
|
+
#: (?String?, ?files: Array[Hash[Symbol, untyped] | Riffer::Messages::User::FilePart]?, ?tags: Hash[(String | Symbol), untyped]) -> Riffer::Agent::Response
|
|
303
327
|
def generate(prompt = nil, files: nil, tags: {})
|
|
304
328
|
Riffer::Agent::Run.generate(agent: self, prompt: prompt, files: files, tags: tags)
|
|
305
329
|
end
|
|
@@ -311,7 +335,7 @@ class Riffer::Agent
|
|
|
311
335
|
# Raises Riffer::ArgumentError if structured output is configured.
|
|
312
336
|
#
|
|
313
337
|
#--
|
|
314
|
-
#: (?String?, ?files: Array[Hash[Symbol, untyped] | Riffer::Messages::FilePart]?, ?tags: Hash[(String | Symbol), untyped]) -> Enumerator[Riffer::StreamEvents::Base, Riffer::Agent::Response]
|
|
338
|
+
#: (?String?, ?files: Array[Hash[Symbol, untyped] | Riffer::Messages::User::FilePart]?, ?tags: Hash[(String | Symbol), untyped]) -> Enumerator[Riffer::StreamEvents::Base, Riffer::Agent::Response]
|
|
315
339
|
def stream(prompt = nil, files: nil, tags: {})
|
|
316
340
|
if @structured_output
|
|
317
341
|
raise Riffer::ArgumentError,
|
data/lib/riffer/config.rb
CHANGED
|
@@ -115,7 +115,7 @@ class Riffer::Config
|
|
|
115
115
|
end
|
|
116
116
|
end
|
|
117
117
|
|
|
118
|
-
# File-attachment-download policy for +Riffer::Messages::FilePart+ URL sources
|
|
118
|
+
# File-attachment-download policy for +Riffer::Messages::User::FilePart+ URL sources
|
|
119
119
|
class Files
|
|
120
120
|
# Allow file attachments to be downloaded to send to providers.
|
|
121
121
|
attr_reader :allow_downloads #: bool # @dynamic allow_downloads
|
|
@@ -15,8 +15,12 @@ class Riffer::Evals::Evaluator
|
|
|
15
15
|
# @rbs self.@instructions: String?
|
|
16
16
|
# @rbs self.@higher_is_better: bool?
|
|
17
17
|
# @rbs self.@judge_model: String?
|
|
18
|
+
# @rbs self.@identifier: String?
|
|
18
19
|
# @rbs @judge: Riffer::Evals::Judge?
|
|
19
20
|
|
|
21
|
+
# The identifier for an anonymous evaluator class.
|
|
22
|
+
DEFAULT_IDENTIFIER = "riffer/judge" #: String
|
|
23
|
+
|
|
20
24
|
class << self
|
|
21
25
|
# Gets or sets the evaluation instructions (criteria and scoring rubric).
|
|
22
26
|
#
|
|
@@ -51,6 +55,28 @@ class Riffer::Evals::Evaluator
|
|
|
51
55
|
|
|
52
56
|
@judge_model = value.to_s
|
|
53
57
|
end
|
|
58
|
+
|
|
59
|
+
# Gets or sets the evaluator identifier, sent as the +agent+ tag on judge
|
|
60
|
+
# calls. Defaults to the snake_cased class name, or DEFAULT_IDENTIFIER for
|
|
61
|
+
# an anonymous class.
|
|
62
|
+
#
|
|
63
|
+
#--
|
|
64
|
+
#: (?String?) -> String
|
|
65
|
+
def identifier(value = nil)
|
|
66
|
+
return @identifier = value.to_s if value
|
|
67
|
+
|
|
68
|
+
@identifier || derived_identifier
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
# Anonymous classes derive an empty identifier.
|
|
74
|
+
#--
|
|
75
|
+
#: () -> String
|
|
76
|
+
def derived_identifier
|
|
77
|
+
derived = Riffer::Helpers::Identifier.for(self)
|
|
78
|
+
derived.empty? ? DEFAULT_IDENTIFIER : derived
|
|
79
|
+
end
|
|
54
80
|
end
|
|
55
81
|
|
|
56
82
|
# Evaluates an input/output pair. The default calls the judge with the
|
|
@@ -94,7 +120,8 @@ class Riffer::Evals::Evaluator
|
|
|
94
120
|
|
|
95
121
|
protected
|
|
96
122
|
|
|
97
|
-
# Returns a Judge instance configured for this evaluator.
|
|
123
|
+
# Returns a Judge instance configured for this evaluator. Its calls carry
|
|
124
|
+
# the +kind+ (+"judge"+) and +agent+ (the evaluator identifier) tags.
|
|
98
125
|
#
|
|
99
126
|
#--
|
|
100
127
|
#: () -> Riffer::Evals::Judge
|
|
@@ -106,7 +133,7 @@ class Riffer::Evals::Evaluator
|
|
|
106
133
|
"No judge model configured. Set judge_model on the evaluator or Riffer.config.evals.judge_model"
|
|
107
134
|
end
|
|
108
135
|
|
|
109
|
-
Riffer::Evals::Judge.new(model: model)
|
|
136
|
+
Riffer::Evals::Judge.new(model: model, tags: { "kind" => "judge", "agent" => self.class.identifier })
|
|
110
137
|
end
|
|
111
138
|
end
|
|
112
139
|
|
data/lib/riffer/evals/judge.rb
CHANGED
|
@@ -30,16 +30,20 @@ class Riffer::Evals::Judge
|
|
|
30
30
|
# The model string (provider/model format).
|
|
31
31
|
attr_reader :model #: String # @dynamic model
|
|
32
32
|
|
|
33
|
+
# The tags passed to the provider on every call.
|
|
34
|
+
attr_reader :tags #: Hash[String, String] # @dynamic tags
|
|
35
|
+
|
|
33
36
|
# Raises Riffer::ArgumentError unless +model+ is "provider/model" format.
|
|
34
37
|
#--
|
|
35
|
-
#: (model: String) -> void
|
|
36
|
-
def initialize(model:)
|
|
38
|
+
#: (model: String, ?tags: Hash[String, String]) -> void
|
|
39
|
+
def initialize(model:, tags: {})
|
|
37
40
|
provider_name, model_name = model.split("/", 2)
|
|
38
41
|
unless [provider_name, model_name].all? { |part| part.is_a?(String) && !part.strip.empty? }
|
|
39
42
|
raise Riffer::ArgumentError, "Invalid model string: #{model}"
|
|
40
43
|
end
|
|
41
44
|
|
|
42
45
|
@model = model
|
|
46
|
+
@tags = tags
|
|
43
47
|
end
|
|
44
48
|
|
|
45
49
|
# Evaluates an input/output pair using the configured LLM.
|
|
@@ -49,12 +53,14 @@ class Riffer::Evals::Judge
|
|
|
49
53
|
system_message = build_system_message(instructions)
|
|
50
54
|
user_message = build_user_message(input: input, output: output, ground_truth: ground_truth)
|
|
51
55
|
|
|
52
|
-
|
|
56
|
+
options = {
|
|
53
57
|
system: system_message,
|
|
54
58
|
prompt: user_message,
|
|
55
59
|
model: model_name,
|
|
56
60
|
tools: [EvaluationTool],
|
|
57
|
-
|
|
61
|
+
} #: Hash[Symbol, untyped]
|
|
62
|
+
options[:tags] = tags unless tags.empty?
|
|
63
|
+
response = provider_instance.generate_text(**options)
|
|
58
64
|
|
|
59
65
|
parse_tool_response(response)
|
|
60
66
|
end
|
|
@@ -111,7 +117,7 @@ class Riffer::Evals::Judge
|
|
|
111
117
|
def parse_tool_response(response)
|
|
112
118
|
tool_call = response.tool_calls.fetch(0) { raise Riffer::Error, "Invalid judge response: no tool call found" }
|
|
113
119
|
|
|
114
|
-
parsed = JSON.parse(tool_call
|
|
120
|
+
parsed = JSON.parse(tool_call.arguments, symbolize_names: true)
|
|
115
121
|
score = parsed[:score]
|
|
116
122
|
reason = parsed[:reason]
|
|
117
123
|
|
|
@@ -40,7 +40,7 @@ class Riffer::Files::Resolver
|
|
|
40
40
|
raise Riffer::TooManyFilesError, "Too many files specified in user message"
|
|
41
41
|
end
|
|
42
42
|
|
|
43
|
-
#: (Riffer::Messages::FilePart) -> void
|
|
43
|
+
#: (Riffer::Messages::User::FilePart) -> void
|
|
44
44
|
def resolve_file!(file)
|
|
45
45
|
delivery = @provider.file_delivery(file)
|
|
46
46
|
if delivery == :unsupported
|
|
@@ -62,7 +62,7 @@ class Riffer::Files::Resolver
|
|
|
62
62
|
end
|
|
63
63
|
end
|
|
64
64
|
|
|
65
|
-
#: (Riffer::Messages::FilePart) -> void
|
|
65
|
+
#: (Riffer::Messages::User::FilePart) -> void
|
|
66
66
|
def verify_inline!(file)
|
|
67
67
|
return unless file.sha256
|
|
68
68
|
|
|
@@ -72,7 +72,7 @@ class Riffer::Files::Resolver
|
|
|
72
72
|
# +cache:+ is false for a :url-delivery provider verifying a sha256 — the
|
|
73
73
|
# request still sends the URL, never the downloaded bytes, so caching them
|
|
74
74
|
# would hold memory nothing reads and let later turns skip re-verifying.
|
|
75
|
-
#: (Riffer::Messages::FilePart, cache: (false | Symbol)) -> void
|
|
75
|
+
#: (Riffer::Messages::User::FilePart, cache: (false | Symbol)) -> void
|
|
76
76
|
def download!(file, cache:)
|
|
77
77
|
raise Riffer::FileDownloadsDisabledError, "File attachments are disabled" unless @config.allow_downloads
|
|
78
78
|
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
# Copying for the nested Hashes and Arrays a configuration object holds.
|
|
5
|
+
module Riffer::Helpers::DeepDup
|
|
6
|
+
extend self
|
|
7
|
+
|
|
8
|
+
# Returns +value+ with every Hash and Array rebuilt, so a copy shares no
|
|
9
|
+
# collection with its source.
|
|
10
|
+
#
|
|
11
|
+
# Anything else is returned as-is, which a Class or a Proc needs: +Class#dup+
|
|
12
|
+
# answers a new anonymous class. One source collection maps to one copy, so
|
|
13
|
+
# references shared within +value+ stay shared in the result — an +:around+
|
|
14
|
+
# guardrail registered under two phases is still one registration afterwards.
|
|
15
|
+
#--
|
|
16
|
+
#: (untyped) -> untyped
|
|
17
|
+
def call(value)
|
|
18
|
+
seen = {} #: Hash[untyped, untyped]
|
|
19
|
+
|
|
20
|
+
rebuild(value, seen.compare_by_identity)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
#--
|
|
26
|
+
#: (untyped, Hash[untyped, untyped]) -> untyped
|
|
27
|
+
def rebuild(value, seen)
|
|
28
|
+
return seen[value] if seen.key?(value)
|
|
29
|
+
|
|
30
|
+
case value
|
|
31
|
+
when Hash
|
|
32
|
+
copy = seen[value] = {}
|
|
33
|
+
value.each { |key, entry| copy[key] = rebuild(entry, seen) }
|
|
34
|
+
copy
|
|
35
|
+
when Array
|
|
36
|
+
copy = seen[value] = []
|
|
37
|
+
value.each { |entry| copy << rebuild(entry, seen) }
|
|
38
|
+
copy
|
|
39
|
+
else value
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
# Represents one block of model reasoning attached to an assistant message.
|
|
5
|
+
# riffer stores and replays a part verbatim rather than interpreting it, so a
|
|
6
|
+
# provider that requires its own thinking blocks back gets them unchanged.
|
|
7
|
+
class Riffer::Messages::Assistant::ReasoningPart
|
|
8
|
+
TYPES = %i[text summary encrypted].freeze #: Array[Symbol]
|
|
9
|
+
|
|
10
|
+
# What the part carries: readable reasoning (+:text+), a provider-condensed
|
|
11
|
+
# digest (+:summary+), or an opaque payload (+:encrypted+).
|
|
12
|
+
attr_reader :type #: Symbol # @dynamic type
|
|
13
|
+
|
|
14
|
+
# The reasoning prose, for +:text+ and +:summary+ parts.
|
|
15
|
+
attr_reader :text #: String? # @dynamic text
|
|
16
|
+
|
|
17
|
+
# The opaque payload, for +:encrypted+ parts.
|
|
18
|
+
attr_reader :data #: String? # @dynamic data
|
|
19
|
+
|
|
20
|
+
# The provider's signature over the part, when it issues one.
|
|
21
|
+
attr_reader :signature #: String? # @dynamic signature
|
|
22
|
+
|
|
23
|
+
# The provider's identifier for the part, when it issues one.
|
|
24
|
+
attr_reader :id #: String? # @dynamic id
|
|
25
|
+
|
|
26
|
+
# The wire format of the part, owned by the provider adapter that produced it
|
|
27
|
+
# (e.g. <tt>"anthropic-claude-v1"</tt>). Adapters replay only the formats they
|
|
28
|
+
# recognize, so it is never validated here.
|
|
29
|
+
attr_reader :format #: String? # @dynamic format
|
|
30
|
+
|
|
31
|
+
# Builds a ReasoningPart from a hash, or returns +part+ unchanged when it is
|
|
32
|
+
# already a ReasoningPart.
|
|
33
|
+
#--
|
|
34
|
+
#: ((Hash[Symbol, untyped] | Riffer::Messages::Assistant::ReasoningPart)) -> Riffer::Messages::Assistant::ReasoningPart
|
|
35
|
+
def self.from_hash(part)
|
|
36
|
+
return part if part.is_a?(Riffer::Messages::Assistant::ReasoningPart)
|
|
37
|
+
|
|
38
|
+
new(
|
|
39
|
+
type: part[:type].to_sym,
|
|
40
|
+
text: part[:text],
|
|
41
|
+
data: part[:data],
|
|
42
|
+
signature: part[:signature],
|
|
43
|
+
id: part[:id],
|
|
44
|
+
format: part[:format],
|
|
45
|
+
)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Raises Riffer::ArgumentError on a +type+ outside TYPES.
|
|
49
|
+
#--
|
|
50
|
+
#: (type: Symbol, ?text: String?, ?data: String?, ?signature: String?, ?id: String?, ?format: String?) -> void
|
|
51
|
+
def initialize(type:, text: nil, data: nil, signature: nil, id: nil, format: nil)
|
|
52
|
+
unless TYPES.include?(type)
|
|
53
|
+
raise Riffer::ArgumentError,
|
|
54
|
+
"type must be one of #{TYPES.inspect}, got #{type.inspect}"
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
@type = type
|
|
58
|
+
@text = text
|
|
59
|
+
@data = data
|
|
60
|
+
@signature = signature
|
|
61
|
+
@id = id
|
|
62
|
+
@format = format
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Serializes the part to a hash, omitting the fields it doesn't carry.
|
|
66
|
+
#
|
|
67
|
+
#--
|
|
68
|
+
#: () -> Hash[Symbol, untyped]
|
|
69
|
+
def to_h
|
|
70
|
+
{ type: type, text: text, data: data, signature: signature, id: id, format: format }.compact
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
#--
|
|
74
|
+
#: (untyped) -> bool
|
|
75
|
+
def ==(other)
|
|
76
|
+
other.is_a?(Riffer::Messages::Assistant::ReasoningPart) && to_h == other.to_h
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
#--
|
|
80
|
+
#: (untyped) -> bool
|
|
81
|
+
def eql?(other)
|
|
82
|
+
self == other
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
#--
|
|
86
|
+
#: () -> Integer
|
|
87
|
+
def hash
|
|
88
|
+
to_h.hash
|
|
89
|
+
end
|
|
90
|
+
end
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# rbs_inline: enabled
|
|
3
|
+
|
|
4
|
+
# Represents one tool invocation the model requested on an assistant message.
|
|
5
|
+
class Riffer::Messages::Assistant::ToolCall
|
|
6
|
+
# The provider's identifier for the call, echoed back on the tool result.
|
|
7
|
+
attr_reader :call_id #: String # @dynamic call_id
|
|
8
|
+
|
|
9
|
+
# The name of the tool to invoke.
|
|
10
|
+
attr_reader :name #: String # @dynamic name
|
|
11
|
+
|
|
12
|
+
# The JSON-encoded arguments, exactly as the provider emitted them.
|
|
13
|
+
attr_reader :arguments #: String # @dynamic arguments
|
|
14
|
+
|
|
15
|
+
# Builds a ToolCall from a hash, or returns +call+ unchanged when it is
|
|
16
|
+
# already a ToolCall. Raises Riffer::ArgumentError when the hash is missing
|
|
17
|
+
# a field.
|
|
18
|
+
#--
|
|
19
|
+
#: ((Hash[Symbol, untyped] | Riffer::Messages::Assistant::ToolCall)) -> Riffer::Messages::Assistant::ToolCall
|
|
20
|
+
def self.from_hash(call)
|
|
21
|
+
return call if call.is_a?(Riffer::Messages::Assistant::ToolCall)
|
|
22
|
+
|
|
23
|
+
if call.values_at(:call_id, :name, :arguments).any?(&:nil?)
|
|
24
|
+
raise Riffer::ArgumentError, "Tool call hash must include :call_id, :name, and :arguments"
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
new(call_id: call[:call_id], name: call[:name], arguments: call[:arguments])
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
#--
|
|
31
|
+
#: (call_id: String, name: String, arguments: String) -> void
|
|
32
|
+
def initialize(call_id:, name:, arguments:)
|
|
33
|
+
@call_id = call_id
|
|
34
|
+
@name = name
|
|
35
|
+
@arguments = arguments
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Serializes the call to a hash.
|
|
39
|
+
#
|
|
40
|
+
#--
|
|
41
|
+
#: () -> Hash[Symbol, untyped]
|
|
42
|
+
def to_h
|
|
43
|
+
{ call_id: call_id, name: name, arguments: arguments }
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
#--
|
|
47
|
+
#: (untyped) -> bool
|
|
48
|
+
def ==(other)
|
|
49
|
+
other.is_a?(Riffer::Messages::Assistant::ToolCall) && to_h == other.to_h
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
#--
|
|
53
|
+
#: (untyped) -> bool
|
|
54
|
+
def eql?(other)
|
|
55
|
+
self == other
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
#--
|
|
59
|
+
#: () -> Integer
|
|
60
|
+
def hash
|
|
61
|
+
to_h.hash
|
|
62
|
+
end
|
|
63
|
+
end
|
|
@@ -4,11 +4,36 @@
|
|
|
4
4
|
# Represents an assistant (LLM) message in a conversation; may include tool
|
|
5
5
|
# calls when the LLM requests tool execution.
|
|
6
6
|
class Riffer::Messages::Assistant < Riffer::Messages::Base
|
|
7
|
-
|
|
7
|
+
# The reasoning part types +reasoning_text+ reads; the rest carry no prose.
|
|
8
|
+
REASONING_TEXT_TYPES = %i[text summary].freeze #: Array[Symbol]
|
|
9
|
+
|
|
10
|
+
# Builds an Assistant message from a hash, or returns +msg+ unchanged when it
|
|
11
|
+
# is already an Assistant message. Raises Riffer::ArgumentError on an invalid
|
|
12
|
+
# tool call, reasoning part, or +finish_reason+.
|
|
13
|
+
#--
|
|
14
|
+
#: ((Hash[Symbol, untyped] | Riffer::Messages::Assistant)) -> Riffer::Messages::Assistant
|
|
15
|
+
def self.from_hash(msg)
|
|
16
|
+
return msg if msg.is_a?(Riffer::Messages::Assistant)
|
|
17
|
+
|
|
18
|
+
new(
|
|
19
|
+
msg[:content],
|
|
20
|
+
id: msg[:id],
|
|
21
|
+
tool_calls: (msg[:tool_calls] || []).map { |tc| Riffer::Messages::Assistant::ToolCall.from_hash(tc) },
|
|
22
|
+
reasoning: (msg[:reasoning] || []).map { |part| Riffer::Messages::Assistant::ReasoningPart.from_hash(part) },
|
|
23
|
+
token_usage: msg[:token_usage] && Riffer::Providers::TokenUsage.from_hash(msg[:token_usage]),
|
|
24
|
+
structured_output: msg[:structured_output],
|
|
25
|
+
finish_reason: msg[:finish_reason]&.to_sym,
|
|
26
|
+
finish_reason_raw: msg[:finish_reason_raw],
|
|
27
|
+
)
|
|
28
|
+
end
|
|
8
29
|
|
|
9
30
|
# Array of tool calls requested by the assistant.
|
|
10
31
|
attr_reader :tool_calls #: Array[Riffer::Messages::Assistant::ToolCall] # @dynamic tool_calls
|
|
11
32
|
|
|
33
|
+
# The model's reasoning blocks for this response, in the order the provider
|
|
34
|
+
# emitted them.
|
|
35
|
+
attr_reader :reasoning #: Array[Riffer::Messages::Assistant::ReasoningPart] # @dynamic reasoning
|
|
36
|
+
|
|
12
37
|
# Token usage data for this response.
|
|
13
38
|
attr_reader :token_usage #: Riffer::Providers::TokenUsage? # @dynamic token_usage
|
|
14
39
|
|
|
@@ -30,6 +55,7 @@ class Riffer::Messages::Assistant < Riffer::Messages::Base
|
|
|
30
55
|
# String,
|
|
31
56
|
# ?id: String?,
|
|
32
57
|
# ?tool_calls: Array[Riffer::Messages::Assistant::ToolCall],
|
|
58
|
+
# ?reasoning: Array[Riffer::Messages::Assistant::ReasoningPart],
|
|
33
59
|
# ?token_usage: Riffer::Providers::TokenUsage?,
|
|
34
60
|
# ?structured_output: Hash[Symbol, untyped]?,
|
|
35
61
|
# ?finish_reason: Symbol?,
|
|
@@ -39,6 +65,7 @@ class Riffer::Messages::Assistant < Riffer::Messages::Base
|
|
|
39
65
|
content,
|
|
40
66
|
id: nil,
|
|
41
67
|
tool_calls: [],
|
|
68
|
+
reasoning: [],
|
|
42
69
|
token_usage: nil,
|
|
43
70
|
structured_output: nil,
|
|
44
71
|
finish_reason: nil,
|
|
@@ -51,6 +78,7 @@ class Riffer::Messages::Assistant < Riffer::Messages::Base
|
|
|
51
78
|
|
|
52
79
|
super(content, id: id)
|
|
53
80
|
@tool_calls = tool_calls
|
|
81
|
+
@reasoning = reasoning
|
|
54
82
|
@token_usage = token_usage
|
|
55
83
|
@structured_output = structured_output
|
|
56
84
|
@finish_reason = finish_reason
|
|
@@ -75,10 +103,29 @@ class Riffer::Messages::Assistant < Riffer::Messages::Base
|
|
|
75
103
|
!@tool_calls.empty?
|
|
76
104
|
end
|
|
77
105
|
|
|
106
|
+
#--
|
|
107
|
+
#: () -> bool
|
|
108
|
+
def reasoning?
|
|
109
|
+
!@reasoning.empty?
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# The readable reasoning across this message's +:text+ and +:summary+ parts,
|
|
113
|
+
# joined by blank lines, or nil when it carries none.
|
|
114
|
+
#--
|
|
115
|
+
#: () -> String?
|
|
116
|
+
def reasoning_text
|
|
117
|
+
texts = reasoning.filter_map { |part| part.text if REASONING_TEXT_TYPES.include?(part.type) }
|
|
118
|
+
texts.empty? ? nil : texts.join("\n\n")
|
|
119
|
+
end
|
|
120
|
+
|
|
78
121
|
#--
|
|
79
122
|
#: (Riffer::Messages::Assistant) -> Riffer::Messages::Assistant
|
|
80
123
|
def +(other)
|
|
81
|
-
self.class.new(
|
|
124
|
+
self.class.new(
|
|
125
|
+
"#{content}\n\n#{other.content}",
|
|
126
|
+
tool_calls: tool_calls + other.tool_calls,
|
|
127
|
+
reasoning: reasoning + other.reasoning,
|
|
128
|
+
)
|
|
82
129
|
end
|
|
83
130
|
|
|
84
131
|
# Converts the message to a hash.
|
|
@@ -89,6 +136,7 @@ class Riffer::Messages::Assistant < Riffer::Messages::Base
|
|
|
89
136
|
hash = { role: role, content: content } #: Hash[Symbol, untyped]
|
|
90
137
|
hash[:id] = id if id
|
|
91
138
|
hash[:tool_calls] = tool_calls.map(&:to_h) unless tool_calls.empty?
|
|
139
|
+
hash[:reasoning] = reasoning.map(&:to_h) if reasoning?
|
|
92
140
|
hash[:token_usage] = token_usage.to_h if token_usage
|
|
93
141
|
hash[:structured_output] = structured_output if structured_output?
|
|
94
142
|
hash[:finish_reason] = finish_reason if finish_reason
|