activeagents-telemetry-ruby_llm 0.2.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf9f1bb148ecabe70a08facf887f60c55dd08f6683370bcb03cb33d1abad0dd1
-  data.tar.gz: 81d06e3973b08667dbb972fe1a013d1be3785b9d9c2b3804a4dfbe5ae67fbaa5
+  metadata.gz: 255ca4c32e9f775c8b2e45ab431ae81ee410fa3002d66dce026ba06d948da4b6
+  data.tar.gz: 1afbdf608524f69045341e10cd7a6f722854cc49977e27ff511cafcc8368fa72
 SHA512:
-  metadata.gz: 999b5249ec348dc4b5280aeddfb2c905af13b29948a85aed5b81ddfdfd2f920fd5d0607b0fd79313f5fad732de271e0db2d1ab23b7ce04e575bf93f90c967040
-  data.tar.gz: 4ec2de8a1cb63605724e7497632b7463ee3e317c133d3bc05cc8131aa8ea1e9b379fdb0da172bf509809bb2ac193c2164bc3780b8844ed32c4c7225220475fb7
+  metadata.gz: 5b316442b6a5bb9044df92b691d6d4dee824d12522c45f4367d252cf7041aeee2be2060ce3a604f9b0e36b814c04e78ef5280646ad67ae7f893c29f0a1799c2b
+  data.tar.gz: 2fe33cb43e2a35659c2112decb662199b461f756e8ef63560e4e7b63151fde0fdcfa1a4f042c6f0419bead7e41787ec43ba51fdbcee58bc4cd1550014a3f56b5

data/README.md

@@ -92,3 +92,44 @@ an explicit `flush!`.
 ```bash
 bundle exec rake test
 ```
+
+## Correlating evaluation traces
+
+Use a separate identity for judge calls and attach stable evaluation identifiers
+without changing the application's default agent resolver:
+
+```ruby
+trace_ids = []
+ActiveAgents::Telemetry::RubyLLM.with_agent(
+  "EvaluationJudge", action: "score",
+  attributes: { "eval.run_id" => run_id, "eval.result_id" => result_id },
+  on_trace: ->(trace) { trace_ids << trace.trace_id },
+  synchronous: true
+) do
+  judge_chat.ask(prompt)
+end
+```
+
+The callback receives each trace the reporter accepted for delivery, so an
+evaluation result can retain the trace ID of that delivery attempt. Acceptance is
+not ingestion: the trace passed the enabled, configured and sampling checks, but
+a delivery that then fails is logged by the reporter rather than announced here,
+and an asynchronous delivery can outlive a process that exits right away. A trace
+dropped by `sample_rate` or by a disabled configuration is never announced.
+`synchronous: true` delivers in the calling thread for this scope only, through
+the same sampling and configuration checks as ordinary delivery; it does not
+mutate the shared async configuration. Normal reporter error logging still
+applies: synchronous delivery does not turn telemetry failures into application
+exceptions. A turn keeps the scope it started under, so a turn left open by a
+pending tool call and closed later by `flush!` still reports with this agent,
+attributes and callback, and a turn that started outside any scope never adopts
+one. Context
+is restored after the block, including when it raises, and nested scopes use
+their own attributes. A callback error is logged by exception class without
+dropping the trace. Attribute redaction and content-capture settings continue to
+apply.
+
+Report publication and trace ingestion are separate operations. Persist each run
+and result ID in the evaluation report and attach the same IDs to its response and
+judge trace attributes. The application chooses whether to publish full report
+content; this API does not enable body capture or upload reports automatically.

data/lib/activeagents/telemetry/ruby_llm/version.rb

@@ -3,7 +3,7 @@
 module ActiveAgents
   module Telemetry
     module RubyLLM
-      VERSION = "0.2.0"
+      VERSION = "0.3.1"
     end
   end
 end

data/lib/activeagents/telemetry/ruby_llm.rb

@@ -43,7 +43,10 @@ module ActiveAgents
 
       DEFAULT_AGENT = { name: "RubyLLM::Chat", action: "chat" }.freeze
 
-      State = Struct.new(:depth, :started_at, :tool_spans, :rounds, :tokens, :chat_key)
+      # Span token keys and the reader that answers each on a RubyLLM 2.x +Tokens+.
+      TOKEN_READERS = { "input" => :input, "output" => :output, "thinking" => :thinking }.freeze
+
+      State = Struct.new(:depth, :started_at, :tool_spans, :rounds, :tokens, :chat_key, :agent)
 
       class << self
         # Subscribes to RubyLLM's instrumentation.
@@ -91,17 +94,30 @@ module ActiveAgents
 
         attr_writer :reporter
 
-        # Attributes traces inside the block to a named agent/action.
-        def with_agent(name, action: "chat")
+        # Attributes traces inside the block to a named agent/action. Correlation
+        # attributes and the callback apply to this scope only. Short-lived
+        # evaluation commands can deliver synchronously without changing the
+        # application's shared reporter configuration.
+        #
+        # A turn keeps the scope it started under, so a turn left open by a
+        # pending tool call and closed later by `flush!` still reports as this
+        # agent, and a turn that started outside any scope never adopts one.
+        # `on_trace` runs only for a trace the reporter accepted, which means
+        # it passed the enabled, configured and sampling checks: a delivery
+        # that then fails is logged by the reporter, not announced here.
+        def with_agent(name, action: "chat", attributes: {}, on_trace: nil, synchronous: false)
           previous = Thread.current[AGENT_KEY]
-          Thread.current[AGENT_KEY] = { name: name, action: action }
+          Thread.current[AGENT_KEY] = {
+            name: name, action: action, attributes: attributes.to_h.transform_keys(&:to_s),
+            on_trace: on_trace, synchronous: synchronous
+          }
           yield
         ensure
           Thread.current[AGENT_KEY] = previous
         end
 
         def state
-          Thread.current[STATE_KEY] ||= State.new(0, nil, [], 0, Span::ZERO_TOKENS.dup, nil)
+          Thread.current[STATE_KEY] ||= State.new(0, nil, [], 0, Span::ZERO_TOKENS.dup, nil, nil)
         end
 
         def clear_state
@@ -126,7 +142,13 @@ module ActiveAgents
             flush! if turn.rounds.positive? && (turn.chat_key != chat_key || turn_expired?(turn))
             turn = state
             turn.chat_key = chat_key
-            turn.started_at ||= Time.now
+            if turn.started_at.nil?
+              turn.started_at = Time.now
+              # Captured once, on the turn's first round, whether or not a scope
+              # is active: a turn that started unscoped stays unscoped even when
+              # a later scope's chat is what flushes it.
+              turn.agent = Thread.current[AGENT_KEY]
+            end
           end
           turn.depth += 1
         end
@@ -161,7 +183,7 @@ module ActiveAgents
         private
 
         def report_turn(payload, turn)
-          agent = Thread.current[AGENT_KEY] || resolve_agent(payload) || DEFAULT_AGENT
+          agent = turn.agent || resolve_agent(payload) || DEFAULT_AGENT
           started_at = turn.started_at || Time.now
           finished_at = Time.now
           error = payload[:exception_object]
@@ -172,12 +194,12 @@ module ActiveAgents
             resource_attributes: configuration.resource_attributes
           )
 
-          root_attributes = {
+          root_attributes = (agent[:attributes] || {}).merge(
             "agent.class" => agent[:name],
             "agent.action" => agent[:action],
             "agent.provider" => payload[:provider].to_s,
             "agent.model" => payload[:model].to_s
-          }
+          )
           root_attributes.merge!(conversation_attributes(payload)) if configuration.capture_bodies?
 
           root = trace.span(
@@ -206,7 +228,14 @@ module ActiveAgents
             trace.add_span(tool_span)
           end
 
-          reporter.report(trace)
+          accepted = agent[:synchronous] ? reporter.report(trace, sync: true) : reporter.report(trace)
+          notify_trace(agent[:on_trace], trace) if accepted
+        end
+
+        def notify_trace(callback, trace)
+          callback&.call(trace)
+        rescue StandardError => e
+          warn "[#{SDK_NAME}] on_trace failed: #{e.class}"
         end
 
         def turn_expired?(turn)
@@ -280,22 +309,39 @@ module ActiveAgents
           text.to_s[0, CONTENT_LIMIT]
         end
 
+        # Tokens for one round. RubyLLM 2.x puts the round's +Tokens+ on the
+        # payload and moves the per-message readers to +message.tokens+; 1.x
+        # has +input_tokens+ and friends on each assistant message and nothing
+        # on the payload. The payload is read first, then the messages the
+        # round added, with either generation's readers.
         def token_totals(payload)
+          tokens = payload_tokens(payload[:tokens]) || message_tokens(payload)
+          tokens["total"] = tokens.values.sum
+          tokens
+        end
+
+        def payload_tokens(tokens)
+          return unless TOKEN_READERS.values.all? { |reader| tokens.respond_to?(reader) }
+
+          TOKEN_READERS.to_h { |key, reader| [ key, tokens.public_send(reader).to_i ] }
+        end
+
+        def message_tokens(payload)
           initial_count = Array(payload[:input_messages]).size
           new_messages = Array(payload[:messages_after])[initial_count..] || []
           assistant_messages = new_messages.select { |message| message.respond_to?(:role) && message.role.to_s == "assistant" }
 
-          tokens = {
-            "input" => sum_tokens(assistant_messages, :input_tokens),
-            "output" => sum_tokens(assistant_messages, :output_tokens),
-            "thinking" => sum_tokens(assistant_messages, :thinking_tokens)
-          }
-          tokens["total"] = tokens.values.sum
-          tokens
+          TOKEN_READERS.to_h { |key, reader| [ key, assistant_messages.sum { |message| message_token_count(message, key, reader) } ] }
         end
 
-        def sum_tokens(messages, method_name)
-          messages.sum { |message| message.respond_to?(method_name) ? message.public_send(method_name).to_i : 0 }
+        # A 1.x message answers +input_tokens+; a 2.x message answers
+        # +tokens.input+.
+        def message_token_count(message, key, reader)
+          legacy_reader = :"#{key}_tokens"
+          return message.public_send(legacy_reader).to_i if message.respond_to?(legacy_reader)
+
+          tokens = message.tokens if message.respond_to?(:tokens)
+          tokens.respond_to?(reader) ? tokens.public_send(reader).to_i : 0
         end
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activeagents-telemetry-ruby_llm
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.1
 platform: ruby
 authors:
 - ActiveAgents
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-08-14 00:00:00.000000000 Z
+date: 2026-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activeagents-telemetry
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement