activeagents-telemetry-ruby_llm 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 74f798054c3b023ac181d541e9ce36d907733511e4abfd1a43d1565cbff09c3e
-  data.tar.gz: c651f96ca0fafc447d5e49090da2f84907fb4a04d33256ad1b87fee1d0e6d0d7
+  metadata.gz: 5abf7d85a5deec9d008170f9c9a3e50d619024b7bad8c1f01eea3eda3d3befc6
+  data.tar.gz: ea10cb9f3cb45700fe8ad0642c72eb3c6e01576c1e4353d274998ead5f3f7243
 SHA512:
-  metadata.gz: 066163ce9683532ee6f2f3220592ede5a85c8a58208bdac182e169c32ccbe5a8c00682e1ac79c936da353399d6aeb024d132253b329d6dcf6e7b2227a0421577
-  data.tar.gz: d1ee5d365abac91ad386ff734095b21aa1b77ef1b0722c4a9d37e2742ab91cc9c9b5898db709f82cda59e7fd1663e224c19fa1691d34c095e6987f4e3d6783b3
+  metadata.gz: 67e5b7d06a3de8510dead200b6ec9a4da44172d24b46c3ea02646cfd4fad1fc3c5acb960798e17a5957f61d8dcd4bd7efc392ae08f8e46ef19b084014e357073
+  data.tar.gz: 16dde44b30870de36415b0958b85eabff49caf639238baf140107e6b2a986effe21bf0eea529c78b5810d90816098cac8c35f09800304c3c5cddd3c7ab6b3c22

data/README.md

@@ -48,7 +48,9 @@ enclosing event, 2.x drives a flat `step until complete?` loop whose rounds
 are siblings with tool calls between them. Rounds are accumulated and flushed
 on the round that ends the turn, so both produce the same trace.
 
-Tool arguments and results are never sent; error messages are truncated.
+Prompts, completions, and tool arguments/results are sent only when the
+configuration's `capture_bodies` is enabled (off by default, truncated to
+4,000 characters); error messages are truncated.
 
 ## Naming the traffic
 
@@ -90,3 +92,44 @@ an explicit `flush!`.
 ```bash
 bundle exec rake test
 ```
+
+## Correlating evaluation traces
+
+Use a separate identity for judge calls and attach stable evaluation identifiers
+without changing the application's default agent resolver:
+
+```ruby
+trace_ids = []
+ActiveAgents::Telemetry::RubyLLM.with_agent(
+  "EvaluationJudge", action: "score",
+  attributes: { "eval.run_id" => run_id, "eval.result_id" => result_id },
+  on_trace: ->(trace) { trace_ids << trace.trace_id },
+  synchronous: true
+) do
+  judge_chat.ask(prompt)
+end
+```
+
+The callback receives each trace the reporter accepted for delivery, so an
+evaluation result can retain the trace ID of that delivery attempt. Acceptance is
+not ingestion: the trace passed the enabled, configured and sampling checks, but
+a delivery that then fails is logged by the reporter rather than announced here,
+and an asynchronous delivery can outlive a process that exits right away. A trace
+dropped by `sample_rate` or by a disabled configuration is never announced.
+`synchronous: true` delivers in the calling thread for this scope only, through
+the same sampling and configuration checks as ordinary delivery; it does not
+mutate the shared async configuration. Normal reporter error logging still
+applies: synchronous delivery does not turn telemetry failures into application
+exceptions. A turn keeps the scope it started under, so a turn left open by a
+pending tool call and closed later by `flush!` still reports with this agent,
+attributes and callback, and a turn that started outside any scope never adopts
+one. Context
+is restored after the block, including when it raises, and nested scopes use
+their own attributes. A callback error is logged by exception class without
+dropping the trace. Attribute redaction and content-capture settings continue to
+apply.
+
+Report publication and trace ingestion are separate operations. Persist each run
+and result ID in the evaluation report and attach the same IDs to its response and
+judge trace attributes. The application chooses whether to publish full report
+content; this API does not enable body capture or upload reports automatically.

data/lib/activeagents/telemetry/ruby_llm/version.rb

@@ -3,7 +3,7 @@
 module ActiveAgents
   module Telemetry
     module RubyLLM
-      VERSION = "0.1.0"
+      VERSION = "0.3.0"
     end
   end
 end

data/lib/activeagents/telemetry/ruby_llm.rb

@@ -27,7 +27,9 @@ module ActiveAgents
     # Tokens are summed per round from the assistant messages that round added,
     # so a repeated event-level count is never double counted.
     #
-    # Tool arguments and results are never sent; error messages are truncated.
+    # Prompts, completions, and tool arguments/results are not sent unless the
+    # configuration's capture_bodies is enabled (they may contain sensitive
+    # data); error messages are truncated.
     module RubyLLM
       AGENT_KEY = :activeagents_telemetry_ruby_llm_agent
       STATE_KEY = :activeagents_telemetry_ruby_llm_state
@@ -36,10 +38,12 @@ module ActiveAgents
       # A turn that never reaches a final round (a halted tool call, or an app
       # driving RubyLLM 2.x's `step` by hand) would otherwise accumulate forever.
       MAX_TURN_SECONDS = 600
+      # Captured prompt/completion/tool content is truncated to this many characters.
+      CONTENT_LIMIT = 4_000
 
       DEFAULT_AGENT = { name: "RubyLLM::Chat", action: "chat" }.freeze
 
-      State = Struct.new(:depth, :started_at, :tool_spans, :rounds, :tokens, :chat_key)
+      State = Struct.new(:depth, :started_at, :tool_spans, :rounds, :tokens, :chat_key, :agent)
 
       class << self
         # Subscribes to RubyLLM's instrumentation.
@@ -87,17 +91,30 @@ module ActiveAgents
 
         attr_writer :reporter
 
-        # Attributes traces inside the block to a named agent/action.
-        def with_agent(name, action: "chat")
+        # Attributes traces inside the block to a named agent/action. Correlation
+        # attributes and the callback apply to this scope only. Short-lived
+        # evaluation commands can deliver synchronously without changing the
+        # application's shared reporter configuration.
+        #
+        # A turn keeps the scope it started under, so a turn left open by a
+        # pending tool call and closed later by `flush!` still reports as this
+        # agent, and a turn that started outside any scope never adopts one.
+        # `on_trace` runs only for a trace the reporter accepted, which means
+        # it passed the enabled, configured and sampling checks: a delivery
+        # that then fails is logged by the reporter, not announced here.
+        def with_agent(name, action: "chat", attributes: {}, on_trace: nil, synchronous: false)
           previous = Thread.current[AGENT_KEY]
-          Thread.current[AGENT_KEY] = { name: name, action: action }
+          Thread.current[AGENT_KEY] = {
+            name: name, action: action, attributes: attributes.to_h.transform_keys(&:to_s),
+            on_trace: on_trace, synchronous: synchronous
+          }
           yield
         ensure
           Thread.current[AGENT_KEY] = previous
         end
 
         def state
-          Thread.current[STATE_KEY] ||= State.new(0, nil, [], 0, Span::ZERO_TOKENS.dup, nil)
+          Thread.current[STATE_KEY] ||= State.new(0, nil, [], 0, Span::ZERO_TOKENS.dup, nil, nil)
         end
 
         def clear_state
@@ -122,7 +139,13 @@ module ActiveAgents
             flush! if turn.rounds.positive? && (turn.chat_key != chat_key || turn_expired?(turn))
             turn = state
             turn.chat_key = chat_key
-            turn.started_at ||= Time.now
+            if turn.started_at.nil?
+              turn.started_at = Time.now
+              # Captured once, on the turn's first round, whether or not a scope
+              # is active: a turn that started unscoped stays unscoped even when
+              # a later scope's chat is what flushes it.
+              turn.agent = Thread.current[AGENT_KEY]
+            end
           end
           turn.depth += 1
         end
@@ -139,11 +162,16 @@ module ActiveAgents
 
         def build_tool_span(payload, started_at, finished_at)
           error = payload[:exception_object]
+          attributes = { "tool.name" => payload[:tool_name].to_s, "tool.call_id" => payload[:tool_call_id].to_s }
+          if configuration.capture_bodies?
+            attributes["tool.arguments"] = tool_io_json(payload[:tool_arguments])
+            attributes["tool.result"] = tool_io_json(payload[:result_content]) unless error
+          end
           span = Span.new(
             "tool.#{payload[:tool_name]}",
             type: "tool",
             start_time: started_at,
-            attributes: { "tool.name" => payload[:tool_name].to_s, "tool.call_id" => payload[:tool_call_id].to_s }
+            attributes: attributes
           )
           span.record_error(error, message_limit: configuration.error_message_limit) if error
           span.finish(at: finished_at)
@@ -152,7 +180,7 @@ module ActiveAgents
         private
 
         def report_turn(payload, turn)
-          agent = Thread.current[AGENT_KEY] || resolve_agent(payload) || DEFAULT_AGENT
+          agent = turn.agent || resolve_agent(payload) || DEFAULT_AGENT
           started_at = turn.started_at || Time.now
           finished_at = Time.now
           error = payload[:exception_object]
@@ -163,14 +191,17 @@ module ActiveAgents
             resource_attributes: configuration.resource_attributes
           )
 
+          root_attributes = (agent[:attributes] || {}).merge(
+            "agent.class" => agent[:name],
+            "agent.action" => agent[:action],
+            "agent.provider" => payload[:provider].to_s,
+            "agent.model" => payload[:model].to_s
+          )
+          root_attributes.merge!(conversation_attributes(payload)) if configuration.capture_bodies?
+
           root = trace.span(
             "#{agent[:name]}.#{agent[:action]}", type: "root", start_time: started_at,
-            attributes: {
-              "agent.class" => agent[:name],
-              "agent.action" => agent[:action],
-              "agent.provider" => payload[:provider].to_s,
-              "agent.model" => payload[:model].to_s
-            }
+            attributes: root_attributes
           )
 
           llm = trace.span(
@@ -194,7 +225,14 @@ module ActiveAgents
             trace.add_span(tool_span)
           end
 
-          reporter.report(trace)
+          accepted = agent[:synchronous] ? reporter.report(trace, sync: true) : reporter.report(trace)
+          notify_trace(agent[:on_trace], trace) if accepted
+        end
+
+        def notify_trace(callback, trace)
+          callback&.call(trace)
+        rescue StandardError => e
+          warn "[#{SDK_NAME}] on_trace failed: #{e.class}"
         end
 
         def turn_expired?(turn)
@@ -215,6 +253,59 @@ module ActiveAgents
           nil
         end
 
+        # The prompt that opened the turn and the answer that closed it — the two
+        # ends a trace is otherwise missing. System instructions are included:
+        # they are the most common cause of a surprising answer.
+        def conversation_attributes(payload)
+          input = Array(payload[:input_messages])
+          new_messages = Array(payload[:messages_after])[input.size..] || []
+
+          attributes = {}
+          if (prompt = last_message_text(input, "user"))
+            attributes["llm.prompt"] = truncate_captured(prompt)
+          end
+          if (instructions = system_instructions(input))
+            attributes["llm.instructions"] = truncate_captured(instructions)
+          end
+          if (completion = last_message_text(new_messages, "assistant"))
+            attributes["llm.completion"] = truncate_captured(completion)
+          end
+          attributes
+        end
+
+        # RubyLLM's `with_instructions` appends by default, so a chat can carry
+        # several system messages and the model sees all of them. Join rather
+        # than taking the last, or an app that layers a base prompt with a
+        # per-request one would report only the fragment.
+        def system_instructions(messages)
+          texts = messages.select do |candidate|
+            candidate.respond_to?(:role) && candidate.role.to_s == "system" &&
+              candidate.respond_to?(:content) && !candidate.content.to_s.empty?
+          end.map { |message| message.content.to_s }
+
+          texts.empty? ? nil : texts.join("\n\n")
+        end
+
+        def last_message_text(messages, role)
+          message = messages.reverse.find do |candidate|
+            candidate.respond_to?(:role) && candidate.role.to_s == role &&
+              candidate.respond_to?(:content) && !candidate.content.to_s.empty?
+          end
+          text = message&.content.to_s
+          text.empty? ? nil : text
+        end
+
+        def tool_io_json(value)
+          json = value.is_a?(String) ? value : JSON.generate(value)
+          truncate_captured(json)
+        rescue StandardError
+          value.inspect[0, CONTENT_LIMIT]
+        end
+
+        def truncate_captured(text)
+          text.to_s[0, CONTENT_LIMIT]
+        end
+
         def token_totals(payload)
           initial_count = Array(payload[:input_messages]).size
           new_messages = Array(payload[:messages_after])[initial_count..] || []

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: activeagents-telemetry-ruby_llm
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - ActiveAgents
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-08-10 00:00:00.000000000 Z
+date: 2026-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activeagents-telemetry
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.1'
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement