activeagents-telemetry-ruby_llm 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf9f1bb148ecabe70a08facf887f60c55dd08f6683370bcb03cb33d1abad0dd1
-  data.tar.gz: 81d06e3973b08667dbb972fe1a013d1be3785b9d9c2b3804a4dfbe5ae67fbaa5
+  metadata.gz: 5abf7d85a5deec9d008170f9c9a3e50d619024b7bad8c1f01eea3eda3d3befc6
+  data.tar.gz: ea10cb9f3cb45700fe8ad0642c72eb3c6e01576c1e4353d274998ead5f3f7243
 SHA512:
-  metadata.gz: 999b5249ec348dc4b5280aeddfb2c905af13b29948a85aed5b81ddfdfd2f920fd5d0607b0fd79313f5fad732de271e0db2d1ab23b7ce04e575bf93f90c967040
-  data.tar.gz: 4ec2de8a1cb63605724e7497632b7463ee3e317c133d3bc05cc8131aa8ea1e9b379fdb0da172bf509809bb2ac193c2164bc3780b8844ed32c4c7225220475fb7
+  metadata.gz: 67e5b7d06a3de8510dead200b6ec9a4da44172d24b46c3ea02646cfd4fad1fc3c5acb960798e17a5957f61d8dcd4bd7efc392ae08f8e46ef19b084014e357073
+  data.tar.gz: 16dde44b30870de36415b0958b85eabff49caf639238baf140107e6b2a986effe21bf0eea529c78b5810d90816098cac8c35f09800304c3c5cddd3c7ab6b3c22

data/README.md

@@ -92,3 +92,44 @@ an explicit `flush!`.
 ```bash
 bundle exec rake test
 ```
+
+## Correlating evaluation traces
+
+Use a separate identity for judge calls and attach stable evaluation identifiers
+without changing the application's default agent resolver:
+
+```ruby
+trace_ids = []
+ActiveAgents::Telemetry::RubyLLM.with_agent(
+  "EvaluationJudge", action: "score",
+  attributes: { "eval.run_id" => run_id, "eval.result_id" => result_id },
+  on_trace: ->(trace) { trace_ids << trace.trace_id },
+  synchronous: true
+) do
+  judge_chat.ask(prompt)
+end
+```
+
+The callback receives each trace the reporter accepted for delivery, so an
+evaluation result can retain the trace ID of that delivery attempt. Acceptance is
+not ingestion: the trace passed the enabled, configured and sampling checks, but
+a delivery that then fails is logged by the reporter rather than announced here,
+and an asynchronous delivery can outlive a process that exits right away. A trace
+dropped by `sample_rate` or by a disabled configuration is never announced.
+`synchronous: true` delivers in the calling thread for this scope only, through
+the same sampling and configuration checks as ordinary delivery; it does not
+mutate the shared async configuration. Normal reporter error logging still
+applies: synchronous delivery does not turn telemetry failures into application
+exceptions. A turn keeps the scope it started under, so a turn left open by a
+pending tool call and closed later by `flush!` still reports with this agent,
+attributes and callback, and a turn that started outside any scope never adopts
+one. Context
+is restored after the block, including when it raises, and nested scopes use
+their own attributes. A callback error is logged by exception class without
+dropping the trace. Attribute redaction and content-capture settings continue to
+apply.
+
+Report publication and trace ingestion are separate operations. Persist each run
+and result ID in the evaluation report and attach the same IDs to its response and
+judge trace attributes. The application chooses whether to publish full report
+content; this API does not enable body capture or upload reports automatically.

data/lib/activeagents/telemetry/ruby_llm/version.rb

@@ -3,7 +3,7 @@
 module ActiveAgents
   module Telemetry
     module RubyLLM
-      VERSION = "0.2.0"
+      VERSION = "0.3.0"
     end
   end
 end

data/lib/activeagents/telemetry/ruby_llm.rb

@@ -43,7 +43,7 @@ module ActiveAgents
 
       DEFAULT_AGENT = { name: "RubyLLM::Chat", action: "chat" }.freeze
 
-      State = Struct.new(:depth, :started_at, :tool_spans, :rounds, :tokens, :chat_key)
+      State = Struct.new(:depth, :started_at, :tool_spans, :rounds, :tokens, :chat_key, :agent)
 
       class << self
         # Subscribes to RubyLLM's instrumentation.
@@ -91,17 +91,30 @@ module ActiveAgents
 
         attr_writer :reporter
 
-        # Attributes traces inside the block to a named agent/action.
-        def with_agent(name, action: "chat")
+        # Attributes traces inside the block to a named agent/action. Correlation
+        # attributes and the callback apply to this scope only. Short-lived
+        # evaluation commands can deliver synchronously without changing the
+        # application's shared reporter configuration.
+        #
+        # A turn keeps the scope it started under, so a turn left open by a
+        # pending tool call and closed later by `flush!` still reports as this
+        # agent, and a turn that started outside any scope never adopts one.
+        # `on_trace` runs only for a trace the reporter accepted, which means
+        # it passed the enabled, configured and sampling checks: a delivery
+        # that then fails is logged by the reporter, not announced here.
+        def with_agent(name, action: "chat", attributes: {}, on_trace: nil, synchronous: false)
           previous = Thread.current[AGENT_KEY]
-          Thread.current[AGENT_KEY] = { name: name, action: action }
+          Thread.current[AGENT_KEY] = {
+            name: name, action: action, attributes: attributes.to_h.transform_keys(&:to_s),
+            on_trace: on_trace, synchronous: synchronous
+          }
           yield
         ensure
           Thread.current[AGENT_KEY] = previous
         end
 
         def state
-          Thread.current[STATE_KEY] ||= State.new(0, nil, [], 0, Span::ZERO_TOKENS.dup, nil)
+          Thread.current[STATE_KEY] ||= State.new(0, nil, [], 0, Span::ZERO_TOKENS.dup, nil, nil)
         end
 
         def clear_state
@@ -126,7 +139,13 @@ module ActiveAgents
             flush! if turn.rounds.positive? && (turn.chat_key != chat_key || turn_expired?(turn))
             turn = state
             turn.chat_key = chat_key
-            turn.started_at ||= Time.now
+            if turn.started_at.nil?
+              turn.started_at = Time.now
+              # Captured once, on the turn's first round, whether or not a scope
+              # is active: a turn that started unscoped stays unscoped even when
+              # a later scope's chat is what flushes it.
+              turn.agent = Thread.current[AGENT_KEY]
+            end
           end
           turn.depth += 1
         end
@@ -161,7 +180,7 @@ module ActiveAgents
         private
 
         def report_turn(payload, turn)
-          agent = Thread.current[AGENT_KEY] || resolve_agent(payload) || DEFAULT_AGENT
+          agent = turn.agent || resolve_agent(payload) || DEFAULT_AGENT
           started_at = turn.started_at || Time.now
           finished_at = Time.now
           error = payload[:exception_object]
@@ -172,12 +191,12 @@ module ActiveAgents
             resource_attributes: configuration.resource_attributes
           )
 
-          root_attributes = {
+          root_attributes = (agent[:attributes] || {}).merge(
             "agent.class" => agent[:name],
             "agent.action" => agent[:action],
             "agent.provider" => payload[:provider].to_s,
             "agent.model" => payload[:model].to_s
-          }
+          )
           root_attributes.merge!(conversation_attributes(payload)) if configuration.capture_bodies?
 
           root = trace.span(
@@ -206,7 +225,14 @@ module ActiveAgents
             trace.add_span(tool_span)
           end
 
-          reporter.report(trace)
+          accepted = agent[:synchronous] ? reporter.report(trace, sync: true) : reporter.report(trace)
+          notify_trace(agent[:on_trace], trace) if accepted
+        end
+
+        def notify_trace(callback, trace)
+          callback&.call(trace)
+        rescue StandardError => e
+          warn "[#{SDK_NAME}] on_trace failed: #{e.class}"
         end
 
         def turn_expired?(turn)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activeagents-telemetry-ruby_llm
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - ActiveAgents
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-08-14 00:00:00.000000000 Z
+date: 2026-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activeagents-telemetry
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement