opentelemetry-instrumentation-ruby_llm 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7dafd7bb3a8ccbb99ed902b457f203d0814a155c0604ea56ef68bdff17030a9
-  data.tar.gz: 21c19ccdf21c94db540dda4d557f3aa97406898f5dc17e4b3a18ee6d42ba9a69
+  metadata.gz: 41a9d9a93e7b336c7256c49918b535ee43de7fdb4f5ecdea78b03e31fad68749
+  data.tar.gz: 02bca3602ca2715d2c5fa4351a9dde621edf8886a521b2ffef9c94a0ce64ee50
 SHA512:
-  metadata.gz: ab14d77a71b7a964b98d92073dd3146f6b2c4917a393b25e524e67fdb101ef14b43a58b4e72384b1c77d5f08f9f21c3bf53e6ff71c2dcb30736fbc24fda2dc4d
-  data.tar.gz: 294d6307dbc72957bde11f5ef11b026d3575bce9794888fe788f73f3c95c06e7fbeb369feda88c5245db92974101e289c8ef0ed36c5263cb38895fd938c54317
+  metadata.gz: 881eec27d1ff083de355052b9bfdce866118c6a3000ec480a4e23cd7aeaf9ce33060b32c48d8f040b3cd26d071e2fa66a774c8097d053ad9616eabdc5a0f79e3
+  data.tar.gz: a582e32144ba0cb3781b3f5a3e1d3aa42e5109abdf0b1fd8044cba527465ad12f4ba783c4c9c305fd226bcfb848cda09214d9db96088fda5f6da83178bfd4fe4

data/.github/workflows/main.yml

@@ -20,6 +20,7 @@ jobs:
           - '3.1'
         gemfile:
           - gemfiles/ruby_llm_1.8.0.gemfile
+          - gemfiles/ruby_llm_1.12.1.gemfile
           - gemfiles/ruby_llm_1_latest.gemfile
 
     env:

data/Appraisals

@@ -9,6 +9,10 @@ appraise "ruby_llm-1.8.0" do
   gem "ruby_llm", "1.8.0"
 end
 
+appraise "ruby_llm-1.12.1" do
+  gem "ruby_llm", "1.12.1"
+end
+
 appraise "ruby_llm-1-latest" do
   gem "ruby_llm", "~> 1.8"
 end

data/Gemfile

@@ -9,8 +9,10 @@ gem "opentelemetry-sdk"
 gem "opentelemetry-exporter-otlp"
 
 group :test do
+  gem "activerecord"
   gem "appraisal", "~> 2.5"
   gem "minitest"
   gem "rake"
+  gem "sqlite3"
   gem "webmock"
 end

data/README.md

@@ -59,6 +59,16 @@ When enabled, the following attributes are added to chat spans:
 > [!WARNING]
 > Captured content may include sensitive or personally identifiable information (PII). Use with caution in production environments.
 
+### Tool result length
+
+Tool call results are recorded on `execute_tool` spans via `gen_ai.tool.call.result`, truncated to 500 characters by default. Adjust the limit with `tool_result_max_length`:
+
+```ruby
+OpenTelemetry::SDK.configure do |c|
+  c.use 'OpenTelemetry::Instrumentation::RubyLLM', tool_result_max_length: 1000
+end
+```
+
 ### Custom attributes
 
 Use `with_otel_attributes` to add arbitrary attributes to the span for each request. This is useful for adding per-request metadata like Langfuse prompt linking or trace-level tags:
@@ -85,15 +95,95 @@ chat.with_otel_attributes(
 
 Attributes persist across calls on the same chat instance and the method returns `self` for chaining.
 
+### Conversation and user tracking
+
+When a chat is a persisted `acts_as_chat` record from
+[RubyLLM's Rails integration](https://rubyllm.com/rails/), its chat spans automatically
+carry `gen_ai.conversation.id` set to the record's id — multi-turn conversations
+correlate (and group as sessions in backends like Langfuse) with no extra code:
+
+```ruby
+chat_record = Chat.create!(model: "gpt-4o-mini")
+chat_record.ask("Hi")
+```
+
+This applies to the modern `acts_as` API (`config.use_new_acts_as = true`). Everywhere
+else — plain `RubyLLM.chat`, the legacy `acts_as` API, or a conversation store outside
+ActiveRecord — set `gen_ai.conversation.id` via `with_otel_attributes` using a real
+conversation/session identifier from your application. An id you set this way always
+wins over the automatic one:
+
+```ruby
+chat.with_otel_attributes("gen_ai.conversation.id" => session.id)
+```
+
+The instrumentation does not generate one for you. Per the
+[GenAI semantic conventions](https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-spans.md),
+when no conversation identifier is available, instrumentations should not populate the
+attribute — a fabricated value such as a random UUID should not be used as a fallback.
+
+You can attach user identity the same way, using the OpenTelemetry
+[`user.*`](https://opentelemetry.io/docs/specs/semconv/registry/attributes/user/) registry
+attributes (the GenAI conventions do not define a user attribute):
+
+```ruby
+chat.with_otel_attributes(
+  "gen_ai.conversation.id" => session.id,
+  "user.id" => current_user.id,
+  "user.email" => current_user.email
+)
+```
+
+### Agent tracing
+
+On `ruby_llm` >= 1.12.1, invoking a `RubyLLM::Agent` subclass wraps the whole run —
+including tool loops and their follow-up completions — in a single `invoke_agent` span:
+
+```ruby
+class ResearchAgent < RubyLLM::Agent
+  model "gpt-4o-mini"
+end
+
+ResearchAgent.new.ask("Find recent papers on prompt caching")
+```
+
+This produces one trace rooted at `invoke_agent ResearchAgent`
+(`gen_ai.operation.name` = `invoke_agent`, `gen_ai.agent.name` = `ResearchAgent`), with
+the chat and tool spans nested beneath it.
+
+When the agent's chat is a persisted `acts_as_chat` record on the modern `acts_as` API,
+the `invoke_agent` span and the chat spans nested beneath it all carry
+`gen_ai.conversation.id` set to the record's id, so multi-turn conversations correlate
+across jobs and requests without any extra code.
+
+With `capture_content` enabled, the `invoke_agent` span also records
+`gen_ai.input.messages` (the conversation history going in) and
+`gen_ai.output.messages` (the final response), so backends that read the trace root
+— like Langfuse — show the run's input and output at the trace level.
+
+Only agent *instances* are wrapped. Class-level entry points that return the chat
+record itself (`ResearchAgent.find(id)`, `ResearchAgent.create!`) bypass the agent
+span — wrap the record with `ResearchAgent.new(chat: record)` to get one.
+
+`with_otel_attributes` works on agents too — attributes are set on the `invoke_agent`
+span (the trace root) and forwarded to the underlying chat:
+
+```ruby
+agent = ResearchAgent.new(chat: chat_record)
+agent.with_otel_attributes("user.id" => current_user.id)
+agent.ask("...")
+```
+
 ## What's traced?
 
 | Feature | Status |
 |---------|--------|
 | Chat completions | Supported |
 | Tool calls | Supported |
+| Agent invocations (`invoke_agent` spans) | Supported (`ruby_llm` >= 1.12.1) |
 | Error handling | Supported |
 | Opt-in input/output content capture | Supported |
-| Conversation tracking (`gen_ai.conversation.id`) | Planned |
+| Conversation tracking (`gen_ai.conversation.id`) | Supported (automatic for persisted `acts_as_chat` records, or set your own id via `with_otel_attributes`) |
 | System instructions capture | Supported (via `capture_content`) |
 | Custom attributes on traces and spans | Supported (via `with_otel_attributes`) |
 | Embeddings | Supported |
@@ -106,6 +196,7 @@ This gem follows the [OpenTelemetry GenAI Semantic Conventions](https://opentele
 This gem is tested against the following `ruby_llm` versions:
 
 - `1.8.0` (minimum supported)
+- `1.12.1` (agent tracing floor — `RubyLLM::Agent` shipped in 1.12.0, but only loads outside Rails from 1.12.1)
 - `~> 1.8` (latest 1.x release)
 
 The Ruby matrix covers Ruby 3.1, 3.2, 3.3, and 3.4.

data/gemfiles/ruby_llm_1.12.1.gemfile

@@ -0,0 +1,18 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "ruby_llm", "1.12.1"
+gem "opentelemetry-sdk"
+gem "opentelemetry-exporter-otlp"
+
+group :test do
+  gem "activerecord"
+  gem "appraisal", "~> 2.5"
+  gem "minitest"
+  gem "rake"
+  gem "sqlite3"
+  gem "webmock"
+end
+
+gemspec path: "../"

data/gemfiles/ruby_llm_1.8.0.gemfile

@@ -7,9 +7,11 @@ gem "opentelemetry-sdk"
 gem "opentelemetry-exporter-otlp"
 
 group :test do
+  gem "activerecord"
   gem "appraisal", "~> 2.5"
   gem "minitest"
   gem "rake"
+  gem "sqlite3"
   gem "webmock"
 end
 

data/gemfiles/ruby_llm_1_latest.gemfile

@@ -7,9 +7,11 @@ gem "opentelemetry-sdk"
 gem "opentelemetry-exporter-otlp"
 
 group :test do
+  gem "activerecord"
   gem "appraisal", "~> 2.5"
   gem "minitest"
   gem "rake"
+  gem "sqlite3"
   gem "webmock"
 end
 

data/lib/opentelemetry/instrumentation/ruby_llm/instrumentation.rb

@@ -5,11 +5,13 @@ module OpenTelemetry
     module RubyLLM
       class Instrumentation < OpenTelemetry::Instrumentation::Base
         MINIMUM_RUBY_LLM_VERSION = "1.8.0"
+        AGENT_MINIMUM_RUBY_LLM_VERSION = "1.12.1"
 
         instrumentation_name "OpenTelemetry::Instrumentation::RubyLLM"
         instrumentation_version VERSION
 
         option :capture_content, default: false, validate: :boolean
+        option :tool_result_max_length, default: 500, validate: :integer
 
         present do
           defined?(::RubyLLM)
@@ -32,10 +34,34 @@ module OpenTelemetry
         end
 
         install do |_config|
+          require_relative "message_formatter"
           require_relative "patches/chat"
           require_relative "patches/embedding"
           ::RubyLLM::Chat.prepend(Patches::Chat)
           ::RubyLLM::Embedding.singleton_class.prepend(Patches::Embedding)
+
+          if Gem::Version.new(::RubyLLM::VERSION) >= Gem::Version.new(AGENT_MINIMUM_RUBY_LLM_VERSION)
+            require_relative "patches/agent"
+            ::RubyLLM::Agent.prepend(Patches::Agent)
+          end
+
+          begin
+            require "active_support/lazy_load_hooks"
+
+            ::ActiveSupport.on_load(:active_record) do
+              require "ruby_llm/active_record/chat_methods"
+              require_relative "patches/chat_methods"
+              ::RubyLLM::ActiveRecord::ChatMethods.prepend(Patches::ChatMethods)
+            rescue LoadError
+              OpenTelemetry.logger.warn(
+                "[OpenTelemetry::Instrumentation::RubyLLM] could not load " \
+                "ruby_llm/active_record/chat_methods; gen_ai.conversation.id " \
+                "will not be set automatically on persisted chat records."
+              )
+            end
+          rescue LoadError
+            nil
+          end
         end
       end
     end

data/lib/opentelemetry/instrumentation/ruby_llm/message_formatter.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module OpenTelemetry
+  module Instrumentation
+    module RubyLLM
+      # Converts `RubyLLM` messages and content into the JSON shape defined by
+      # the GenAI semantic conventions for input/output messages and system
+      # instructions:
+      #
+      #   https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-input-messages.json
+      #   https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-output-messages.json
+      #   https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-system-instructions.json
+      #
+      # Kept separate from the `RubyLLM::Chat` patch so the formatting logic
+      # does not pollute the patched class.
+      module MessageFormatter
+        def self.format_input_messages(messages)
+          messages.map { |m| format_message(m) }.to_json
+        end
+
+        def self.format_output_messages(messages)
+          messages.map { |m| format_message(m) }.to_json
+        end
+
+        def self.format_system_instructions(messages)
+          messages.flat_map { |m| format_content(m.content) }.to_json
+        end
+
+        private_class_method def self.format_message(message)
+          msg = { role: message.role.to_s, parts: [] }
+
+          if message.content
+            msg[:parts].concat(format_content(message.content))
+          end
+
+          if message.tool_calls&.any?
+            message.tool_calls.each_value do |tc|
+              msg[:parts] << { type: "tool_call", id: tc.id, name: tc.name, arguments: tc.arguments }
+            end
+          end
+
+          msg[:tool_call_id] = message.tool_call_id if message.tool_call_id
+
+          msg
+        end
+
+        # Maps a `RubyLLM::Content`/`RubyLLM::Content::Raw` onto an array of
+        # GenAI message parts.
+        private_class_method def self.format_content(content)
+          # `RubyLLM::Content::Raw` was added in ruby_llm 1.9.0, so guard the
+          # constant rather than referencing it in a `case`/`when`.
+          if defined?(::RubyLLM::Content::Raw) && content.is_a?(::RubyLLM::Content::Raw)
+            # Serialize the provider-specific payload to JSON so consumers
+            # (e.g. Langfuse) render it as readable text rather than
+            # `[object Object]`.
+            [{ type: "raw", content: content.value.to_json }]
+          elsif content.is_a?(::RubyLLM::Content)
+            parts = []
+            parts << { type: "text", content: content.text } unless content.text.nil?
+            content.attachments.each do |attachment|
+              parts << format_attachment(attachment)
+            end
+            parts
+          else
+            [{ type: "text", content: content.to_s }]
+          end
+        end
+
+        # Maps a `RubyLLM::Attachment` onto a GenAI message part.
+        private_class_method def self.format_attachment(attachment)
+          part = { modality: attachment_modality(attachment) }
+          part[:mime_type] = attachment.mime_type if attachment.mime_type
+
+          if attachment.url?
+            part[:type] = "uri"
+            part[:uri] = attachment.source.to_s
+          else
+            part[:type] = "blob"
+            part[:content] = attachment.source.to_s
+          end
+
+          part
+        end
+
+        # Maps a `RubyLLM::Attachment#type` onto a GenAI modality.
+        private_class_method def self.attachment_modality(attachment)
+          case attachment.type
+          when :image then "image"
+          when :video then "video"
+          when :audio then "audio"
+          else "document"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/opentelemetry/instrumentation/ruby_llm/patches/agent.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module OpenTelemetry
+  module Instrumentation
+    module RubyLLM
+      module Patches
+        module Agent
+          def with_otel_attributes(attributes)
+            @otel_attributes = attributes
+            llm_chat.with_otel_attributes(attributes)
+            self
+          end
+
+          def ask(...)
+            in_invoke_agent_span { super }
+          end
+
+          def say(...)
+            in_invoke_agent_span { super }
+          end
+
+          def complete(...)
+            in_invoke_agent_span { super }
+          end
+
+          private
+
+          def in_invoke_agent_span
+            agent_name = self.class.name
+            attributes = { "gen_ai.operation.name" => "invoke_agent" }
+            attributes["gen_ai.agent.name"] = agent_name if agent_name
+            conversation_id = llm_chat.otel_conversation_id if llm_chat.respond_to?(:otel_conversation_id)
+            attributes["gen_ai.conversation.id"] = conversation_id if conversation_id
+
+            span_name = agent_name ? "invoke_agent #{agent_name}" : "invoke_agent"
+
+            tracer.in_span(span_name, attributes: attributes, kind: OpenTelemetry::Trace::SpanKind::INTERNAL) do |span|
+              result = yield
+              capture_messages(span)
+              result
+            rescue => e
+              span.set_attribute("error.type", e.class.name)
+              raise
+            ensure
+              set_custom_attributes(span)
+            end
+          end
+
+          def capture_messages(span)
+            return unless capture_content?
+
+            messages = llm_chat.messages
+            return if messages.empty?
+
+            input_messages = messages[0..-2].reject { |m| m.role == :system }
+            span.set_attribute("gen_ai.input.messages", MessageFormatter.format_input_messages(input_messages))
+            span.set_attribute("gen_ai.output.messages", MessageFormatter.format_output_messages([messages.last]))
+          end
+
+          def llm_chat
+            chat.respond_to?(:to_llm) ? chat.to_llm : chat
+          end
+
+          def capture_content?
+            env_value = ENV["OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"]
+            return env_value.to_s.strip.casecmp("true").zero? unless env_value.nil?
+
+            RubyLLM::Instrumentation.instance.config[:capture_content]
+          end
+
+          def set_custom_attributes(span)
+            @otel_attributes&.each { |key, value| span.set_attribute(key, value.respond_to?(:call) ? value.call : value) }
+          rescue => e
+            OpenTelemetry.handle_error(exception: e)
+          end
+
+          def tracer
+            RubyLLM::Instrumentation.instance.tracer
+          end
+        end
+      end
+    end
+  end
+end

data/lib/opentelemetry/instrumentation/ruby_llm/patches/chat.rb

@@ -5,6 +5,13 @@ module OpenTelemetry
     module RubyLLM
       module Patches
         module Chat
+          attr_writer :otel_conversation_id
+
+          def otel_conversation_id
+            id = @otel_attributes&.[]("gen_ai.conversation.id") || @otel_conversation_id
+            id.respond_to?(:call) ? id.call : id
+          end
+
           def with_otel_attributes(attributes)
             @otel_attributes = attributes
             self
@@ -19,6 +26,8 @@ module OpenTelemetry
               "gen_ai.provider.name" => provider,
               "gen_ai.request.model" => model_id,
             }
+            conversation_id = otel_conversation_id
+            attributes["gen_ai.conversation.id"] = conversation_id if conversation_id
             # Per GenAI semconv: set `gen_ai.request.stream` if and only if
             # the request is streaming. Absence means non-streaming.
             attributes["gen_ai.request.stream"] = true if block_given?
@@ -55,11 +64,11 @@ module OpenTelemetry
                   input_messages = @messages[0..-2].reject { |m| m.role == :system }
 
                   unless system_messages.empty?
-                    span.set_attribute("gen_ai.system_instructions", format_system_instructions(system_messages))
+                    span.set_attribute("gen_ai.system_instructions", MessageFormatter.format_system_instructions(system_messages))
                   end
 
-                  span.set_attribute("gen_ai.input.messages", format_messages(input_messages))
-                  span.set_attribute("gen_ai.output.messages", format_messages([response]))
+                  span.set_attribute("gen_ai.input.messages", MessageFormatter.format_input_messages(input_messages))
+                  span.set_attribute("gen_ai.output.messages", MessageFormatter.format_output_messages([response]))
                 end
               end
 
@@ -91,7 +100,7 @@ module OpenTelemetry
 
               # `RubyLLM::Tool::Halt#to_s` returns `@content.to_s`, so a single
               # `to_s` covers both the Halt and plain-result cases.
-              span.set_attribute("gen_ai.tool.call.result", result.to_s[0..500])
+              span.set_attribute("gen_ai.tool.call.result", result.to_s[0, tool_result_max_length])
 
               result
             end
@@ -106,30 +115,8 @@ module OpenTelemetry
             RubyLLM::Instrumentation.instance.config[:capture_content]
           end
 
-          def format_messages(messages)
-            messages.map { |m| format_message(m) }.to_json
-          end
-
-          def format_message(message)
-            msg = { role: message.role.to_s, parts: [] }
-
-            if message.content
-              msg[:parts] << { type: "text", content: message.content.to_s }
-            end
-
-            if message.tool_calls&.any?
-              message.tool_calls.each_value do |tc|
-                msg[:parts] << { type: "tool_call", id: tc.id, name: tc.name, arguments: tc.arguments }
-              end
-            end
-
-            msg[:tool_call_id] = message.tool_call_id if message.tool_call_id
-
-            msg
-          end
-
-          def format_system_instructions(system_messages)
-            system_messages.map { |m| { type: "text", content: m.content.to_s } }.to_json
+          def tool_result_max_length
+            RubyLLM::Instrumentation.instance.config[:tool_result_max_length]
           end
 
           def tracer

data/lib/opentelemetry/instrumentation/ruby_llm/patches/chat_methods.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module OpenTelemetry
+  module Instrumentation
+    module RubyLLM
+      module Patches
+        module ChatMethods
+          def to_llm(...)
+            chat = super
+            chat.otel_conversation_id = id.to_s if persisted?
+            chat
+          end
+        end
+      end
+    end
+  end
+end

data/lib/opentelemetry/instrumentation/ruby_llm/version.rb

@@ -3,7 +3,7 @@
 module OpenTelemetry
   module Instrumentation
     module RubyLLM
-      VERSION = "0.6.0"
+      VERSION = "0.7.0"
     end
   end
 end