opentelemetry-instrumentation-ruby_llm 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e85590961ea3d4ac30828dbc035d692040a2a66758ba6dab9decab2114af3f3
-  data.tar.gz: a8a701fa288ec93109bfb6a4d3fdcc5962c40abc98d9a9269d12a9e430848f78
+  metadata.gz: dfb17d4d74d31d2a53792683e0d5d070cd25e9160ee3c5018c0a1dbfb5bbeb4b
+  data.tar.gz: 91c3b7ac277d43ddf80a6fc49e2228d8005bf40f9321cd10cd5ac2539c96dfc7
 SHA512:
-  metadata.gz: 60672653bad15a0e24a97a60ea94433f82df8300370987f656f135a5eb1339b731afbf3644dddbe4a867b3f9970d2b757e06ca7862a4b7d0d36798dff3f9aa58
-  data.tar.gz: d4536fbcba4b6439df8c8a4a30198072eea8cc61265ee1572a5c2b0dbd6f775260381d727d06ebc13604d755dc98f44f24f7ee552d218bb50b2695aec6ad3873
+  metadata.gz: df479838c253aa695c1b36b393b4f7eac31dd041a581df3d04d9f9e9ca1253d026b8582b32d2cdf398e812d53459ef0c012406654342d37a9ae45c45dc68c5e7
+  data.tar.gz: 9c18a05253193ce23af8bc0f342cd726778c6daccff7d1010602a63b36f20d07eb13852ea6f01d6e40bf2439236fbf4b495e32e144e701d521aaeec18972f6a4

data/.gitignore

@@ -1 +1,2 @@
 *.gem
+Gemfile.lock

data/README.md

@@ -30,6 +30,35 @@ OpenTelemetry::SDK.configure do |c|
 end
 ```
 
+## Configuration
+
+### Content capture
+
+By default, message content is **not captured**. To enable it:
+
+```ruby
+OpenTelemetry::SDK.configure do |c|
+  c.use 'OpenTelemetry::Instrumentation::RubyLLM', capture_content: true
+end
+```
+
+Or set the environment variable:
+
+```bash
+export OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true
+```
+
+When enabled, the following attributes are added to chat spans:
+
+| Attribute | Description |
+|-----------|-------------|
+| `gen_ai.system_instructions` | System instructions provided via `with_instructions` |
+| `gen_ai.input.messages` | Input messages sent to the model |
+| `gen_ai.output.messages` | Final output messages from the model |
+
+> [!WARNING]
+> Captured content may include sensitive or personally identifiable information (PII). Use with caution in production environments.
+
 ## What's traced?
 
 | Feature | Status |
@@ -37,9 +66,10 @@ end
 | Chat completions | Supported |
 | Tool calls | Supported |
 | Error handling | Supported |
+| Opt-in input/output content capture | Supported |
 | Conversation tracking (`gen_ai.conversation.id`) | Planned |
-| Opt-in input/output content capture | Planned |
 | System instructions capture | Planned |
+| Custom attributes on traces and spans | Planned |
 | Embeddings | Planned |
 | Streaming | Planned |
 

data/example/trace_demonstration_with_langfuse.rb

@@ -26,7 +26,7 @@ OpenTelemetry::SDK.configure do |c|
       )
     )
   )
-  c.use "OpenTelemetry::Instrumentation::RubyLLM"
+  c.use "OpenTelemetry::Instrumentation::RubyLLM", capture_content: true
 end
 
 RubyLLM.configure do |c|
@@ -35,6 +35,7 @@ RubyLLM.configure do |c|
 end
 
 chat = RubyLLM.chat
+chat.with_instructions("You are a helpful assistant that provides concise answers.")
 response = chat.ask("What is the meaning of life?")
 puts "\nResponse: #{response.content}"
 

data/example/trace_demonstration_with_langfuse_and_tools.rb

@@ -26,7 +26,7 @@ OpenTelemetry::SDK.configure do |c|
       )
     )
   )
-  c.use "OpenTelemetry::Instrumentation::RubyLLM"
+  c.use "OpenTelemetry::Instrumentation::RubyLLM", capture_content: true
 end
 
 RubyLLM.configure do |c|
@@ -44,9 +44,12 @@ class Calculator < RubyLLM::Tool
 end
 
 chat = RubyLLM.chat
+chat.with_instructions("You are a helpful assistant that provides concise answers.")
 chat.with_tool(Calculator)
 response = chat.ask("Use the calculator tool to compute 123 * 456")
 puts "\nResponse: #{response.content}"
+response = chat.ask("Use the tool again to compute 789 + 1011")
+puts "\nResponse: #{response.content}"
 
 # This line is only necessary in short-lived scripts. In a long-running application, spans will be flushed automatically.
 OpenTelemetry.tracer_provider.force_flush

data/lib/opentelemetry/instrumentation/ruby_llm/instrumentation.rb

@@ -7,6 +7,8 @@ module OpenTelemetry
         instrumentation_name "OpenTelemetry::Instrumentation::RubyLLM"
         instrumentation_version VERSION
 
+        option :capture_content, default: false, validate: :boolean
+
         present do
           defined?(::RubyLLM)
         end

data/lib/opentelemetry/instrumentation/ruby_llm/patches/chat.rb

@@ -5,7 +5,7 @@ module OpenTelemetry
     module RubyLLM
       module Patches
         module Chat
-          def ask(message, &block)
+          def complete(&)
             provider = @model&.provider || "unknown"
             model_id = @model&.id || "unknown"
 
@@ -18,22 +18,34 @@ module OpenTelemetry
             tracer.in_span("chat #{model_id}", attributes: attributes, kind: OpenTelemetry::Trace::SpanKind::CLIENT) do |span|
               begin
                 result = super
-
-                if @messages.last
-                  response = @messages.last
-                  span.set_attribute("gen_ai.response.model", response.model_id) if response.model_id
-                  span.set_attribute("gen_ai.usage.input_tokens", response.input_tokens) if response.input_tokens
-                  span.set_attribute("gen_ai.usage.output_tokens", response.output_tokens) if response.output_tokens
-                  span.set_attribute("gen_ai.request.temperature", @temperature) if @temperature
-                end
-
-                result
               rescue => e
                 span.record_exception(e)
                 span.status = OpenTelemetry::Trace::Status.error(e.message)
                 span.set_attribute("error.type", e.class.name)
                 raise
               end
+
+              if @messages.last
+                response = @messages.last
+                span.set_attribute("gen_ai.response.model", response.model_id) if response.model_id
+                span.set_attribute("gen_ai.usage.input_tokens", response.input_tokens) if response.input_tokens
+                span.set_attribute("gen_ai.usage.output_tokens", response.output_tokens) if response.output_tokens
+                span.set_attribute("gen_ai.request.temperature", @temperature) if @temperature
+
+                if capture_content?
+                  system_messages = @messages.select { |m| m.role == :system }
+                  input_messages = @messages[0..-2].reject { |m| m.role == :system }
+
+                  unless system_messages.empty?
+                    span.set_attribute("gen_ai.system_instructions", format_system_instructions(system_messages))
+                  end
+
+                  span.set_attribute("gen_ai.input.messages", format_messages(input_messages))
+                  span.set_attribute("gen_ai.output.messages", format_messages([response]))
+                end
+              end
+
+              result
             end
           rescue StandardError => e
             OpenTelemetry.handle_error(exception: e)
@@ -61,6 +73,39 @@ module OpenTelemetry
 
           private
 
+          def capture_content?
+            env_value = ENV["OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"]
+            return env_value.to_s.strip.casecmp("true").zero? unless env_value.nil?
+
+            RubyLLM::Instrumentation.instance.config[:capture_content]
+          end
+
+          def format_messages(messages)
+            messages.map { |m| format_message(m) }.to_json
+          end
+
+          def format_message(message)
+            msg = { role: message.role.to_s, parts: [] }
+
+            if message.content
+              msg[:parts] << { type: "text", content: message.content.to_s }
+            end
+
+            if message.tool_calls&.any?
+              message.tool_calls.each_value do |tc|
+                msg[:parts] << { type: "tool_call", id: tc.id, name: tc.name, arguments: tc.arguments }
+              end
+            end
+
+            msg[:tool_call_id] = message.tool_call_id if message.tool_call_id
+
+            msg
+          end
+
+          def format_system_instructions(system_messages)
+            system_messages.map { |m| { type: "text", content: m.content.to_s } }.to_json
+          end
+
           def tracer
             RubyLLM::Instrumentation.instance.tracer
           end

data/lib/opentelemetry/instrumentation/ruby_llm/version.rb

@@ -3,7 +3,7 @@
 module OpenTelemetry
   module Instrumentation
     module RubyLLM
-      VERSION = "0.1.0"
+      VERSION = "0.3.0"
     end
   end
 end

data/test/instrumentation_test.rb

@@ -67,7 +67,7 @@ class InstrumentationTest < Minitest::Test
     assert_equal OpenTelemetry::Trace::Status::ERROR, span.status.code
   end
 
-  def test_ask_still_works_when_instrumentation_fails
+  def test_complete_still_works_when_instrumentation_fails
     stub_request(:post, "https://api.openai.com/v1/chat/completions")
       .to_return(
         status: 200,
@@ -92,6 +92,39 @@ class InstrumentationTest < Minitest::Test
     assert_equal "Hello!", response.content
   end
 
+  def test_instruments_complete_called_directly
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        status: 200,
+        headers: { "Content-Type" => "application/json" },
+        body: {
+          id: "chatcmpl-123",
+          object: "chat.completion",
+          model: "gpt-4o-mini",
+          choices: [{
+            index: 0,
+            message: { role: "assistant", content: "Hello, world!" },
+            finish_reason: "stop"
+          }],
+          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
+        }.to_json
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.add_message(role: :user, content: "Hi")
+    chat.complete
+
+    spans = EXPORTER.finished_spans
+    assert_equal 1, spans.length
+
+    span = spans.first
+    assert_equal "chat gpt-4o-mini", span.name
+    assert_equal "chat", span.attributes["gen_ai.operation.name"]
+    assert_equal "openai", span.attributes["gen_ai.provider.name"]
+    assert_equal 10, span.attributes["gen_ai.usage.input_tokens"]
+    assert_equal 5, span.attributes["gen_ai.usage.output_tokens"]
+  end
+
   def test_creates_span_for_tool_call
     calculator = Class.new(RubyLLM::Tool) do
       def self.name = "calculator"
@@ -155,7 +188,7 @@ class InstrumentationTest < Minitest::Test
     chat_spans = spans.select { |s| s.name.include?("chat ") }
 
     assert_equal 1, tool_spans.length
-    assert_equal 1, chat_spans.length
+    assert_equal 2, chat_spans.length
 
     tool_span = tool_spans.first
     assert_equal OpenTelemetry::Trace::SpanKind::INTERNAL, tool_span.kind
@@ -228,4 +261,108 @@ class InstrumentationTest < Minitest::Test
     response = chat.ask("What is 2+2?")
     assert_equal "The answer is 4", response.content
   end
+
+  def test_does_not_capture_content_by_default
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        status: 200,
+        headers: { "Content-Type" => "application/json" },
+        body: {
+          id: "chatcmpl-123",
+          object: "chat.completion",
+          model: "gpt-4o-mini",
+          choices: [{
+            index: 0,
+            message: { role: "assistant", content: "Hello, world!" },
+            finish_reason: "stop"
+          }],
+          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
+        }.to_json
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.with_instructions("You are helpful")
+    chat.ask("Hi")
+
+    span = EXPORTER.finished_spans.first
+    assert_nil span.attributes["gen_ai.system_instructions"]
+    assert_nil span.attributes["gen_ai.input.messages"]
+    assert_nil span.attributes["gen_ai.output.messages"]
+  end
+
+  def test_captures_content_when_enabled
+    OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = true
+
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        status: 200,
+        headers: { "Content-Type" => "application/json" },
+        body: {
+          id: "chatcmpl-123",
+          object: "chat.completion",
+          model: "gpt-4o-mini",
+          choices: [{
+            index: 0,
+            message: { role: "assistant", content: "Hello, world!" },
+            finish_reason: "stop"
+          }],
+          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
+        }.to_json
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.with_instructions("You are helpful")
+    chat.ask("Hi")
+
+    span = EXPORTER.finished_spans.first
+
+    system_instructions = JSON.parse(span.attributes["gen_ai.system_instructions"])
+    assert_equal [{ "type" => "text", "content" => "You are helpful" }], system_instructions
+
+    input_messages = JSON.parse(span.attributes["gen_ai.input.messages"])
+    assert_equal 1, input_messages.length
+    assert_equal "user", input_messages[0]["role"]
+    assert_equal [{ "type" => "text", "content" => "Hi" }], input_messages[0]["parts"]
+
+    output_messages = JSON.parse(span.attributes["gen_ai.output.messages"])
+    assert_equal 1, output_messages.length
+    assert_equal "assistant", output_messages[0]["role"]
+    assert_equal [{ "type" => "text", "content" => "Hello, world!" }], output_messages[0]["parts"]
+  ensure
+    OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = false
+  end
+
+  def test_captures_content_when_enabled_via_env_var
+    ENV["OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"] = "true"
+
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        status: 200,
+        headers: { "Content-Type" => "application/json" },
+        body: {
+          id: "chatcmpl-123",
+          object: "chat.completion",
+          model: "gpt-4o-mini",
+          choices: [{
+            index: 0,
+            message: { role: "assistant", content: "Hello, world!" },
+            finish_reason: "stop"
+          }],
+          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
+        }.to_json
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.ask("Hi")
+
+    span = EXPORTER.finished_spans.first
+
+    input_messages = JSON.parse(span.attributes["gen_ai.input.messages"])
+    assert_equal "user", input_messages[0]["role"]
+
+    output_messages = JSON.parse(span.attributes["gen_ai.output.messages"])
+    assert_equal "assistant", output_messages[0]["role"]
+  ensure
+    ENV.delete("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT")
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: opentelemetry-instrumentation-ruby_llm
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Clarissa Borges
@@ -51,7 +51,6 @@ files:
 - CODEOWNERS
 - CODE_OF_CONDUCT.md
 - Gemfile
-- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile

data/Gemfile.lock

@@ -1,105 +0,0 @@
-PATH
-  remote: .
-  specs:
-    opentelemetry-instrumentation-ruby_llm (0.1.0)
-      opentelemetry-api (~> 1.0)
-      opentelemetry-instrumentation-base (~> 0.23)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    addressable (2.8.8)
-      public_suffix (>= 2.0.2, < 8.0)
-    base64 (0.3.0)
-    bigdecimal (4.0.1)
-    crack (1.0.1)
-      bigdecimal
-      rexml
-    event_stream_parser (1.0.0)
-    faraday (2.14.0)
-      faraday-net_http (>= 2.0, < 3.5)
-      json
-      logger
-    faraday-multipart (1.2.0)
-      multipart-post (~> 2.0)
-    faraday-net_http (3.4.2)
-      net-http (~> 0.5)
-    faraday-retry (2.4.0)
-      faraday (~> 2.0)
-    google-protobuf (4.33.4)
-      bigdecimal
-      rake (>= 13)
-    google-protobuf (4.33.4-arm64-darwin)
-      bigdecimal
-      rake (>= 13)
-    googleapis-common-protos-types (1.22.0)
-      google-protobuf (~> 4.26)
-    hashdiff (1.2.1)
-    json (2.18.0)
-    logger (1.7.0)
-    marcel (1.1.0)
-    minitest (6.0.1)
-      prism (~> 1.5)
-    multipart-post (2.4.1)
-    net-http (0.9.1)
-      uri (>= 0.11.1)
-    opentelemetry-api (1.7.0)
-    opentelemetry-common (0.23.0)
-      opentelemetry-api (~> 1.0)
-    opentelemetry-exporter-otlp (0.31.1)
-      google-protobuf (>= 3.18)
-      googleapis-common-protos-types (~> 1.3)
-      opentelemetry-api (~> 1.1)
-      opentelemetry-common (~> 0.20)
-      opentelemetry-sdk (~> 1.10)
-      opentelemetry-semantic_conventions
-    opentelemetry-instrumentation-base (0.25.0)
-      opentelemetry-api (~> 1.7)
-      opentelemetry-common (~> 0.21)
-      opentelemetry-registry (~> 0.1)
-    opentelemetry-registry (0.4.0)
-      opentelemetry-api (~> 1.1)
-    opentelemetry-sdk (1.10.0)
-      opentelemetry-api (~> 1.1)
-      opentelemetry-common (~> 0.20)
-      opentelemetry-registry (~> 0.2)
-      opentelemetry-semantic_conventions
-    opentelemetry-semantic_conventions (1.36.0)
-      opentelemetry-api (~> 1.0)
-    prism (1.9.0)
-    public_suffix (7.0.2)
-    rake (13.3.1)
-    rexml (3.4.4)
-    ruby_llm (1.11.0)
-      base64
-      event_stream_parser (~> 1)
-      faraday (>= 1.10.0)
-      faraday-multipart (>= 1)
-      faraday-net_http (>= 1)
-      faraday-retry (>= 1)
-      marcel (~> 1.0)
-      ruby_llm-schema (~> 0.2.1)
-      zeitwerk (~> 2)
-    ruby_llm-schema (0.2.5)
-    uri (1.1.1)
-    webmock (3.26.1)
-      addressable (>= 2.8.0)
-      crack (>= 0.3.2)
-      hashdiff (>= 0.4.0, < 2.0.0)
-    zeitwerk (2.7.4)
-
-PLATFORMS
-  arm64-darwin-24
-  ruby
-
-DEPENDENCIES
-  minitest
-  opentelemetry-exporter-otlp
-  opentelemetry-instrumentation-ruby_llm!
-  opentelemetry-sdk
-  rake
-  ruby_llm
-  webmock
-
-BUNDLED WITH
-   2.7.1