activeagents-telemetry-ruby_llm 0.3.0 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5abf7d85a5deec9d008170f9c9a3e50d619024b7bad8c1f01eea3eda3d3befc6
-  data.tar.gz: ea10cb9f3cb45700fe8ad0642c72eb3c6e01576c1e4353d274998ead5f3f7243
+  metadata.gz: e97d04ad7846320b7c233f247ce4ecad0ea2f20dba9574a73d2c3f93d3a47bfd
+  data.tar.gz: 50e301417298577aad66eb5d8423689d2c24742e5cc91be7a5fd55f518695c22
 SHA512:
-  metadata.gz: 67e5b7d06a3de8510dead200b6ec9a4da44172d24b46c3ea02646cfd4fad1fc3c5acb960798e17a5957f61d8dcd4bd7efc392ae08f8e46ef19b084014e357073
-  data.tar.gz: 16dde44b30870de36415b0958b85eabff49caf639238baf140107e6b2a986effe21bf0eea529c78b5810d90816098cac8c35f09800304c3c5cddd3c7ab6b3c22
+  metadata.gz: 759dd7ce43f5c74c4eef4003ec72e2254e618523a132fcd0e7d8c249416fb2c8a4d96170d2984d7a6c352ca95ff4240d1beff259d8d96b410807e02836660199
+  data.tar.gz: 2e49c0ec5f861e55fc42304ea30e3f27e5bcefe8043e841d54cb2bcb8bc39c95870793baa87d6128006ada7e5b8e12dc3aeeaa83ad5fd5d4b277c663b64e2d3c

data/README.md

@@ -124,10 +124,66 @@ exceptions. A turn keeps the scope it started under, so a turn left open by a
 pending tool call and closed later by `flush!` still reports with this agent,
 attributes and callback, and a turn that started outside any scope never adopts
 one. Context
-is restored after the block, including when it raises, and nested scopes use
-their own attributes. A callback error is logged by exception class without
-dropping the trace. Attribute redaction and content-capture settings continue to
-apply.
+is restored after the block, including when it raises. A callback error is
+logged by exception class without dropping the trace. Attribute redaction and
+content-capture settings continue to apply.
+
+### Nested scopes
+
+Scopes compose, so an evaluation can wrap code that names its own agent
+without losing what the evaluation attached. When a `with_agent` block opens
+inside another one:
+
+| Setting | Nested scope inside a plain scope | Nested scope inside a `pin: true` scope |
+|---------|-----------------------------------|------------------------------------------|
+| `attributes` | Merged: enclosing first, nested keys win on collision | Merged the same way |
+| `on_trace` | Inherited unless the nested call passes its own | The pinned scope's callback stays in force |
+| `synchronous` | Inherited unless the nested call passes its own (`nil`, the default, inherits; an explicit `false` overrides an inherited `true`) | The pinned scope's setting stays in force |
+| `name` / `action` | Replaced by the nested scope | The pinned scope's identity stays in force |
+
+A pinned scope owns the turn's identity: while it is active, a nested
+`with_agent` may only add attributes, and it stays pinned however deep the
+nesting goes. The previous scope is restored exactly on exit, including when
+the block raises. The `agent.*` identity keys are always written last, so an
+attribute with one of those names never overrides the reported agent.
+
+```ruby
+ActiveAgents::Telemetry::RubyLLM.with_agent(
+  "SupportAgent", action: "replay",
+  attributes: { "eval.run_id" => run_id, "eval.result_id" => result_id },
+  on_trace: ->(trace) { result.trace_id = trace.trace_id },
+  synchronous: true, pin: true
+) do
+  # The code under test names its own scope around the actual call. The
+  # trace reports as SupportAgent.replay with eval.* and sparkle.role, the
+  # evaluation's on_trace receives it, and it is delivered before this
+  # block returns.
+  ActiveAgents::Telemetry::RubyLLM.with_agent("SupportAgent", action: "respond",
+    attributes: { "sparkle.role" => "assistant" }) do
+    chat.ask(prompt)
+  end
+end
+```
+
+### Correlating with ActiveAgent evaluations
+
+`ActiveAgent::Evals::Correlation` (activeagent >= 1.6.3) takes a `tracer:`
+that it calls as `tracer.call(name, action:, attributes:, on_trace:) { ... }`
+around each replay, and expects `on_trace` to receive something answering
+`#trace_id`. `correlation_tracer` returns exactly that lambda, opening a
+pinned, synchronous `with_agent` scope with the arguments it is given and
+returning the block's value:
+
+```ruby
+ActiveAgent::Evals::Correlation.new(
+  agent_name: "SupportAgent",
+  tracer: ActiveAgents::Telemetry::RubyLLM.correlation_tracer
+)
+```
+
+`correlation_tracer(synchronous: false)` or `correlation_tracer(pin: false)`
+relaxes either default. This gem does not depend on activeagent; the only
+contract shared with it is the lambda's signature.
 
 Report publication and trace ingestion are separate operations. Persist each run
 and result ID in the evaluation report and attach the same IDs to its response and

data/lib/activeagents/telemetry/ruby_llm/version.rb

@@ -3,7 +3,7 @@
 module ActiveAgents
   module Telemetry
     module RubyLLM
-      VERSION = "0.3.0"
+      VERSION = "0.3.2"
     end
   end
 end

data/lib/activeagents/telemetry/ruby_llm.rb

@@ -43,6 +43,9 @@ module ActiveAgents
 
       DEFAULT_AGENT = { name: "RubyLLM::Chat", action: "chat" }.freeze
 
+      # Span token keys and the reader that answers each on a RubyLLM 2.x +Tokens+.
+      TOKEN_READERS = { "input" => :input, "output" => :output, "thinking" => :thinking }.freeze
+
       State = Struct.new(:depth, :started_at, :tool_spans, :rounds, :tokens, :chat_key, :agent)
 
       class << self
@@ -102,17 +105,71 @@ module ActiveAgents
         # `on_trace` runs only for a trace the reporter accepted, which means
         # it passed the enabled, configured and sampling checks: a delivery
         # that then fails is logged by the reporter, not announced here.
-        def with_agent(name, action: "chat", attributes: {}, on_trace: nil, synchronous: false)
+        #
+        # Scopes compose. Inside an enclosing scope, a nested call merges its
+        # `attributes` onto the enclosing ones (a nested key wins on
+        # collision) and inherits `on_trace` and `synchronous` unless it
+        # passes its own; `synchronous: nil`, the default, means "inherit,
+        # else false", so an explicit `synchronous: false` still overrides an
+        # inherited `true`. Without `pin`, the nested scope replaces `name`
+        # and `action`, so a call site that names its own agent keeps doing
+        # so under an evaluation's scope. A scope opened with `pin: true` owns
+        # the turn's identity: while it is active a nested `with_agent` may
+        # only add attributes, and the pinned scope's `name`, `action`,
+        # `on_trace` and `synchronous` stay in force whatever the nested call
+        # passes. An evaluation replay pins its scope so the code under test,
+        # which opens its own scope around the actual `chat.ask`, cannot
+        # detach the trace from the evaluation's attributes, callback and
+        # synchronous delivery. The previous scope is restored on exit,
+        # including when the block raises.
+        #
+        # @param synchronous [Boolean, nil] nil inherits the enclosing scope's
+        #   setting, or false at the top level.
+        # @param pin [Boolean] whether this scope keeps its identity, callback
+        #   and delivery mode against nested scopes.
+        def with_agent(name, action: "chat", attributes: {}, on_trace: nil, synchronous: nil, pin: false)
           previous = Thread.current[AGENT_KEY]
-          Thread.current[AGENT_KEY] = {
-            name: name, action: action, attributes: attributes.to_h.transform_keys(&:to_s),
-            on_trace: on_trace, synchronous: synchronous
-          }
+          Thread.current[AGENT_KEY] = compose_scope(
+            previous, name: name, action: action, attributes: attributes,
+            on_trace: on_trace, synchronous: synchronous, pin: pin
+          )
           yield
         ensure
           Thread.current[AGENT_KEY] = previous
         end
 
+        # A `tracer:` for `ActiveAgent::Evals::Correlation` (activeagent >=
+        # 1.6.3), which wraps each evaluation replay in
+        # `tracer.call(name, action:, attributes:, on_trace:) { ... }` and
+        # expects `on_trace` to receive something answering `#trace_id`. The
+        # returned lambda opens a pinned, synchronous `with_agent` scope with
+        # exactly those arguments, so the replayed code's own `with_agent`
+        # around `chat.ask` can add attributes but cannot replace the
+        # evaluation's identity, drop its `on_trace` or turn delivery
+        # asynchronous, and the trace id the callback receives is the one
+        # the reporter delivered before the replay returns. The block's value
+        # is returned. There is no dependency on activeagent: the contract is
+        # only the lambda's signature.
+        #
+        #   ActiveAgent::Evals::Correlation.new(
+        #     agent_name: "SupportAgent",
+        #     tracer: ActiveAgents::Telemetry::RubyLLM.correlation_tracer
+        #   )
+        #
+        # @param synchronous [Boolean] whether the scope delivers in the
+        #   calling thread; true keeps the trace id available when the
+        #   replay returns and before a short-lived process exits.
+        # @param pin [Boolean] whether nested scopes may only add attributes.
+        # @return [Proc] `->(name, action:, attributes:, on_trace:, &block)`
+        def correlation_tracer(synchronous: true, pin: true)
+          lambda do |name, action: "chat", attributes: {}, on_trace: nil, &block|
+            with_agent(
+              name, action: action, attributes: attributes, on_trace: on_trace,
+              synchronous: synchronous, pin: pin, &block
+            )
+          end
+        end
+
         def state
           Thread.current[STATE_KEY] ||= State.new(0, nil, [], 0, Span::ZERO_TOKENS.dup, nil, nil)
         end
@@ -179,6 +236,27 @@ module ActiveAgents
 
         private
 
+        # The scope a nested `with_agent` installs, given the enclosing one.
+        # Attributes always merge with the nested keys winning; the rest
+        # depends on whether the enclosing scope is pinned.
+        def compose_scope(enclosing, name:, action:, attributes:, on_trace:, synchronous:, pin:)
+          own = attributes.to_h.transform_keys(&:to_s)
+          if enclosing.nil?
+            return { name: name, action: action, attributes: own, on_trace: on_trace,
+                     synchronous: synchronous ? true : false, pin: pin ? true : false }
+          end
+
+          merged = (enclosing[:attributes] || {}).merge(own)
+          return enclosing.merge(attributes: merged) if enclosing[:pin]
+
+          {
+            name: name, action: action, attributes: merged,
+            on_trace: on_trace.nil? ? enclosing[:on_trace] : on_trace,
+            synchronous: synchronous.nil? ? enclosing[:synchronous] : synchronous,
+            pin: pin ? true : false
+          }
+        end
+
         def report_turn(payload, turn)
           agent = turn.agent || resolve_agent(payload) || DEFAULT_AGENT
           started_at = turn.started_at || Time.now
@@ -306,22 +384,39 @@ module ActiveAgents
           text.to_s[0, CONTENT_LIMIT]
         end
 
+        # Tokens for one round. RubyLLM 2.x puts the round's +Tokens+ on the
+        # payload and moves the per-message readers to +message.tokens+; 1.x
+        # has +input_tokens+ and friends on each assistant message and nothing
+        # on the payload. The payload is read first, then the messages the
+        # round added, with either generation's readers.
         def token_totals(payload)
+          tokens = payload_tokens(payload[:tokens]) || message_tokens(payload)
+          tokens["total"] = tokens.values.sum
+          tokens
+        end
+
+        def payload_tokens(tokens)
+          return unless TOKEN_READERS.values.all? { |reader| tokens.respond_to?(reader) }
+
+          TOKEN_READERS.to_h { |key, reader| [ key, tokens.public_send(reader).to_i ] }
+        end
+
+        def message_tokens(payload)
           initial_count = Array(payload[:input_messages]).size
           new_messages = Array(payload[:messages_after])[initial_count..] || []
           assistant_messages = new_messages.select { |message| message.respond_to?(:role) && message.role.to_s == "assistant" }
 
-          tokens = {
-            "input" => sum_tokens(assistant_messages, :input_tokens),
-            "output" => sum_tokens(assistant_messages, :output_tokens),
-            "thinking" => sum_tokens(assistant_messages, :thinking_tokens)
-          }
-          tokens["total"] = tokens.values.sum
-          tokens
+          TOKEN_READERS.to_h { |key, reader| [ key, assistant_messages.sum { |message| message_token_count(message, key, reader) } ] }
         end
 
-        def sum_tokens(messages, method_name)
-          messages.sum { |message| message.respond_to?(method_name) ? message.public_send(method_name).to_i : 0 }
+        # A 1.x message answers +input_tokens+; a 2.x message answers
+        # +tokens.input+.
+        def message_token_count(message, key, reader)
+          legacy_reader = :"#{key}_tokens"
+          return message.public_send(legacy_reader).to_i if message.respond_to?(legacy_reader)
+
+          tokens = message.tokens if message.respond_to?(:tokens)
+          tokens.respond_to?(reader) ? tokens.public_send(reader).to_i : 0
         end
       end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activeagents-telemetry-ruby_llm
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.2
 platform: ruby
 authors:
 - ActiveAgents
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-13 00:00:00.000000000 Z
+date: 2026-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activeagents-telemetry