opentelemetry-instrumentation-ruby_llm 0.6.0 → 0.7.0

data/test/instrumentation_test.rb

@@ -1,11 +1,14 @@
 require "test_helper"
 
 class InstrumentationTest < Minitest::Test
+  include ChatCompletionStubs
+
   def setup
     EXPORTER.reset
 
     RubyLLM.configure do |c|
       c.openai_api_key = "fake-key-for-testing"
+      c.anthropic_api_key = "fake-key-for-testing"
     end
   end
 
@@ -30,29 +33,12 @@ class InstrumentationTest < Minitest::Test
     assert_equal "1.8.0", OpenTelemetry::Instrumentation::RubyLLM::Instrumentation::MINIMUM_RUBY_LLM_VERSION
   end
 
+  def test_agent_minimum_ruby_llm_version_is_pinned_at_1_12_1
+    assert_equal "1.12.1", OpenTelemetry::Instrumentation::RubyLLM::Instrumentation::AGENT_MINIMUM_RUBY_LLM_VERSION
+  end
+
   def test_creates_span_with_attributes
-    stub_request(:post, "https://api.openai.com/v1/chat/completions")
-      .to_return(
-        status: 200,
-        headers: { "Content-Type" => "application/json" },
-        body: {
-          id: "chatcmpl-123",
-          object: "chat.completion",
-          model: "gpt-4o-mini",
-          choices: [
-            {
-              index: 0,
-              message: { role: "assistant", content: "Hello, world!" },
-              finish_reason: "stop"
-            }
-          ],
-          usage: {
-            prompt_tokens: 10,
-            completion_tokens: 5,
-            total_tokens: 15
-          }
-        }.to_json
-      )
+    stub_chat_completion
 
     chat = RubyLLM.chat(model: "gpt-4o-mini")
     chat.ask("Hi")
@@ -73,17 +59,9 @@ class InstrumentationTest < Minitest::Test
   end
 
   def test_marks_streaming_chat_requests
-    stub_request(:post, "https://api.openai.com/v1/chat/completions")
-      .to_return(
-        status: 200,
-        headers: { "Content-Type" => "application/json" },
-        body: {
-          id: "chatcmpl-123",
-          model: "gpt-4o-mini",
-          choices: [{ index: 0, message: { role: "assistant", content: "Hi" }, finish_reason: "stop" }],
-          usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }
-        }.to_json
-      )
+    stub_chat_completion(
+      chat_completion_body(content: "Hi", usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 })
+    )
 
     chat = RubyLLM.chat(model: "gpt-4o-mini")
     chat.ask("Hi") { |_chunk| }
@@ -92,28 +70,58 @@ class InstrumentationTest < Minitest::Test
     assert_equal true, span.attributes["gen_ai.request.stream"]
   end
 
-  def test_records_prompt_cache_tokens
-    # RubyLLM's OpenAI provider maps `cached_tokens` ← `cache_read_tokens(usage)`
-    # and `cache_creation_tokens` ← `cache_write_tokens(usage)`, both surfaced
-    # on `Message#cached_tokens` / `Message#cache_creation_tokens`.
-    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+  def test_records_openai_prompt_cache_read_tokens
+    # OpenAI exposes only `cached_tokens` via `prompt_tokens_details.cached_tokens`.
+    # Its provider in ruby_llm does not surface a `cache_creation_tokens` value
+    # until 1.15.0 (we only assert the cache-read attribute here).
+    # The accessor itself was added in ruby_llm 1.9.0.
+    skip "cached_tokens accessor not available before ruby_llm 1.9.0" unless RubyLLM::Message.instance_methods.include?(:cached_tokens)
+    stub_chat_completion(
+      chat_completion_body(
+        content: "Hello!",
+        usage: {
+          prompt_tokens: 100,
+          completion_tokens: 5,
+          total_tokens: 105,
+          prompt_tokens_details: { cached_tokens: 75 }
+        }
+      )
+    )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.ask("Hi")
+
+    span = EXPORTER.finished_spans.first
+    assert_equal 75, span.attributes["gen_ai.usage.cache_read.input_tokens"]
+    assert_equal 0, span.attributes["gen_ai.usage.cache_creation.input_tokens"]
+  end
+
+  def test_records_anthropic_prompt_cache_tokens
+    # Anthropic's provider surfaces both `cached_tokens` (via
+    # `cache_read_input_tokens`) and `cache_creation_tokens` (via
+    # `cache_creation_input_tokens`). Accessors were added in ruby_llm 1.9.0.
+    skip "cache token accessors not available before ruby_llm 1.9.0" unless RubyLLM::Message.instance_methods.include?(:cache_creation_tokens)
+    stub_request(:post, "https://api.anthropic.com/v1/messages")
       .to_return(
         status: 200,
         headers: { "Content-Type" => "application/json" },
         body: {
-          id: "chatcmpl-cache",
-          model: "gpt-4o-mini",
-          choices: [{ index: 0, message: { role: "assistant", content: "Hello!" }, finish_reason: "stop" }],
+          id: "msg_cache",
+          type: "message",
+          role: "assistant",
+          model: "claude-3-5-sonnet-20241022",
+          content: [{ type: "text", text: "Hello!" }],
+          stop_reason: "end_turn",
           usage: {
-            prompt_tokens: 100,
-            completion_tokens: 5,
-            total_tokens: 105,
-            prompt_tokens_details: { cached_tokens: 75, cache_write_tokens: 20 }
+            input_tokens: 100,
+            output_tokens: 5,
+            cache_read_input_tokens: 75,
+            cache_creation_input_tokens: 20
           }
         }.to_json
       )
 
-    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat = RubyLLM.chat(model: "claude-3-5-sonnet-20241022")
     chat.ask("Hi")
 
     span = EXPORTER.finished_spans.first
@@ -140,22 +148,7 @@ class InstrumentationTest < Minitest::Test
   end
 
   def test_instruments_complete_called_directly
-    stub_request(:post, "https://api.openai.com/v1/chat/completions")
-      .to_return(
-        status: 200,
-        headers: { "Content-Type" => "application/json" },
-        body: {
-          id: "chatcmpl-123",
-          object: "chat.completion",
-          model: "gpt-4o-mini",
-          choices: [{
-            index: 0,
-            message: { role: "assistant", content: "Hello, world!" },
-            finish_reason: "stop"
-          }],
-          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
-        }.to_json
-      )
+    stub_chat_completion
 
     chat = RubyLLM.chat(model: "gpt-4o-mini")
     chat.add_message(role: :user, content: "Hi")
@@ -183,47 +176,20 @@ class InstrumentationTest < Minitest::Test
       end
     end
 
-    stub_request(:post, "https://api.openai.com/v1/chat/completions")
-      .to_return(
-        {
-          status: 200,
-          headers: { "Content-Type" => "application/json" },
-          body: {
-            id: "chatcmpl-123",
-            object: "chat.completion",
-            model: "gpt-4o-mini",
-            choices: [{
-              index: 0,
-              message: {
-                role: "assistant",
-                content: nil,
-                tool_calls: [{
-                  id: "call_abc123",
-                  type: "function",
-                  function: { name: "calculator", arguments: '{"expression":"2+2"}' }
-                }]
-              },
-              finish_reason: "tool_calls"
-            }],
-            usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
-          }.to_json
-        },
-        {
-          status: 200,
-          headers: { "Content-Type" => "application/json" },
-          body: {
-            id: "chatcmpl-456",
-            object: "chat.completion",
-            model: "gpt-4o-mini",
-            choices: [{
-              index: 0,
-              message: { role: "assistant", content: "The answer is 4" },
-              finish_reason: "stop"
-            }],
-            usage: { prompt_tokens: 20, completion_tokens: 5, total_tokens: 25 }
-          }.to_json
-        }
+    stub_chat_completion(
+      chat_completion_body(
+        content: nil,
+        tool_calls: [{
+          id: "call_abc123",
+          type: "function",
+          function: { name: "calculator", arguments: '{"expression":"2+2"}' }
+        }]
+      ),
+      chat_completion_body(
+        content: "The answer is 4",
+        usage: { prompt_tokens: 20, completion_tokens: 5, total_tokens: 25 }
       )
+    )
 
     chat = RubyLLM.chat(model: "gpt-4o-mini")
     chat.with_tool(calculator)
@@ -249,6 +215,41 @@ class InstrumentationTest < Minitest::Test
     assert_equal "function", tool_span.attributes["gen_ai.tool.type"]
   end
 
+  def test_truncates_tool_result_to_configured_max_length
+    long_value = "x" * 1000
+    echo = Class.new(RubyLLM::Tool) do
+      def self.name = "echo"
+      description "Echoes a long string"
+
+      define_method(:execute) { long_value }
+    end
+
+    stub_chat_completion(
+      chat_completion_body(
+        content: nil,
+        tool_calls: [{
+          id: "call_echo",
+          type: "function",
+          function: { name: "echo", arguments: "{}" }
+        }]
+      ),
+      chat_completion_body(
+        content: "done",
+        usage: { prompt_tokens: 20, completion_tokens: 5, total_tokens: 25 }
+      )
+    )
+
+    OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:tool_result_max_length] = 700
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini").with_tool(echo)
+    chat.ask("echo please")
+
+    tool_span = EXPORTER.finished_spans.find { |s| s.name.start_with?("execute_tool ") }
+    assert_equal "x" * 700, tool_span.attributes["gen_ai.tool.call.result"]
+  ensure
+    OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:tool_result_max_length] = 500
+  end
+
   def test_records_error_when_tool_raises
     boom = Class.new(RubyLLM::Tool) do
       def self.name = "boom"
@@ -259,29 +260,17 @@ class InstrumentationTest < Minitest::Test
       end
     end
 
-    stub_request(:post, "https://api.openai.com/v1/chat/completions")
-      .to_return(
-        status: 200,
-        headers: { "Content-Type" => "application/json" },
-        body: {
-          id: "chatcmpl-boom",
-          model: "gpt-4o-mini",
-          choices: [{
-            index: 0,
-            message: {
-              role: "assistant",
-              content: nil,
-              tool_calls: [{
-                id: "call_x",
-                type: "function",
-                function: { name: "boom", arguments: "{}" }
-              }]
-            },
-            finish_reason: "tool_calls"
-          }],
-          usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }
-        }.to_json
+    stub_chat_completion(
+      chat_completion_body(
+        content: nil,
+        tool_calls: [{
+          id: "call_x",
+          type: "function",
+          function: { name: "boom", arguments: "{}" }
+        }],
+        usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }
       )
+    )
 
     chat = RubyLLM.chat(model: "gpt-4o-mini").with_tool(boom)
     assert_raises(ArgumentError) { chat.ask("trigger") }
@@ -292,22 +281,7 @@ class InstrumentationTest < Minitest::Test
   end
 
   def test_does_not_capture_content_by_default
-    stub_request(:post, "https://api.openai.com/v1/chat/completions")
-      .to_return(
-        status: 200,
-        headers: { "Content-Type" => "application/json" },
-        body: {
-          id: "chatcmpl-123",
-          object: "chat.completion",
-          model: "gpt-4o-mini",
-          choices: [{
-            index: 0,
-            message: { role: "assistant", content: "Hello, world!" },
-            finish_reason: "stop"
-          }],
-          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
-        }.to_json
-      )
+    stub_chat_completion
 
     chat = RubyLLM.chat(model: "gpt-4o-mini")
     chat.with_instructions("You are helpful")
@@ -322,22 +296,7 @@ class InstrumentationTest < Minitest::Test
   def test_captures_content_when_enabled
     OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = true
 
-    stub_request(:post, "https://api.openai.com/v1/chat/completions")
-      .to_return(
-        status: 200,
-        headers: { "Content-Type" => "application/json" },
-        body: {
-          id: "chatcmpl-123",
-          object: "chat.completion",
-          model: "gpt-4o-mini",
-          choices: [{
-            index: 0,
-            message: { role: "assistant", content: "Hello, world!" },
-            finish_reason: "stop"
-          }],
-          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
-        }.to_json
-      )
+    stub_chat_completion
 
     chat = RubyLLM.chat(model: "gpt-4o-mini")
     chat.with_instructions("You are helpful")
@@ -409,22 +368,7 @@ class InstrumentationTest < Minitest::Test
   end
 
   def test_with_otel_attributes_sets_span_attributes
-    stub_request(:post, "https://api.openai.com/v1/chat/completions")
-      .to_return(
-        status: 200,
-        headers: { "Content-Type" => "application/json" },
-        body: {
-          id: "chatcmpl-123",
-          object: "chat.completion",
-          model: "gpt-4o-mini",
-          choices: [{
-            index: 0,
-            message: { role: "assistant", content: "Hello!" },
-            finish_reason: "stop"
-          }],
-          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
-        }.to_json
-      )
+    stub_chat_completion(chat_completion_body(content: "Hello!"))
 
     chat = RubyLLM.chat(model: "gpt-4o-mini")
     chat.with_otel_attributes(
@@ -439,22 +383,7 @@ class InstrumentationTest < Minitest::Test
   end
 
   def test_with_otel_attributes_returns_self_for_chaining
-    stub_request(:post, "https://api.openai.com/v1/chat/completions")
-      .to_return(
-        status: 200,
-        headers: { "Content-Type" => "application/json" },
-        body: {
-          id: "chatcmpl-123",
-          object: "chat.completion",
-          model: "gpt-4o-mini",
-          choices: [{
-            index: 0,
-            message: { role: "assistant", content: "Hello!" },
-            finish_reason: "stop"
-          }],
-          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
-        }.to_json
-      )
+    stub_chat_completion(chat_completion_body(content: "Hello!"))
 
     chat = RubyLLM.chat(model: "gpt-4o-mini")
     result = chat.with_otel_attributes("custom.category" => "test")
@@ -463,22 +392,7 @@ class InstrumentationTest < Minitest::Test
   end
 
   def test_with_otel_attributes_evaluates_callables
-    stub_request(:post, "https://api.openai.com/v1/chat/completions")
-      .to_return(
-        status: 200,
-        headers: { "Content-Type" => "application/json" },
-        body: {
-          id: "chatcmpl-123",
-          object: "chat.completion",
-          model: "gpt-4o-mini",
-          choices: [{
-            index: 0,
-            message: { role: "assistant", content: "Hello!" },
-            finish_reason: "stop"
-          }],
-          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
-        }.to_json
-      )
+    stub_chat_completion(chat_completion_body(content: "Hello!"))
 
     chat = RubyLLM.chat(model: "gpt-4o-mini")
     chat.with_otel_attributes(
@@ -493,22 +407,7 @@ class InstrumentationTest < Minitest::Test
   end
 
   def test_works_without_otel_attributes
-    stub_request(:post, "https://api.openai.com/v1/chat/completions")
-      .to_return(
-        status: 200,
-        headers: { "Content-Type" => "application/json" },
-        body: {
-          id: "chatcmpl-123",
-          object: "chat.completion",
-          model: "gpt-4o-mini",
-          choices: [{
-            index: 0,
-            message: { role: "assistant", content: "Hello!" },
-            finish_reason: "stop"
-          }],
-          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
-        }.to_json
-      )
+    stub_chat_completion(chat_completion_body(content: "Hello!"))
 
     chat = RubyLLM.chat(model: "gpt-4o-mini")
     response = chat.ask("Hi")
@@ -516,25 +415,84 @@ class InstrumentationTest < Minitest::Test
     assert_equal "Hello!", response.content
   end
 
+  def test_captures_ruby_llm_content_with_attachments
+    OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = true
+
+    stub_chat_completion(chat_completion_body(content: "A cat."))
+
+    content = RubyLLM::Content.new("What is this?", "https://example.com/cat.png")
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.add_message(role: :user, content: content)
+    chat.complete
+
+    span = EXPORTER.finished_spans.first
+    input_messages = JSON.parse(span.attributes["gen_ai.input.messages"])
+    assert_equal(
+      [
+        { "type" => "text", "content" => "What is this?" },
+        {
+          "type" => "uri",
+          "modality" => "image",
+          "mime_type" => "image/png",
+          "uri" => "https://example.com/cat.png"
+        }
+      ],
+      input_messages[0]["parts"]
+    )
+  ensure
+    OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = false
+  end
+
+  def test_captures_ruby_llm_content_raw
+    skip "RubyLLM::Content::Raw not available before ruby_llm 1.9.0" unless defined?(RubyLLM::Content::Raw)
+    OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = true
+
+    stub_chat_completion(chat_completion_body(content: "Acknowledged."))
+
+    raw = RubyLLM::Content::Raw.new([{ type: "text", text: "raw payload" }])
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.add_message(role: :user, content: raw)
+    chat.complete
+
+    span = EXPORTER.finished_spans.first
+    input_messages = JSON.parse(span.attributes["gen_ai.input.messages"])
+    assert_equal(
+      [{ "type" => "raw", "content" => [{ "type" => "text", "text" => "raw payload" }].to_json }],
+      input_messages[0]["parts"]
+    )
+  ensure
+    OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = false
+  end
+
+  def test_captures_ruby_llm_content_raw_system_instructions
+    skip "RubyLLM::Content::Raw not available before ruby_llm 1.9.0" unless defined?(RubyLLM::Content::Raw)
+    OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = true
+
+    stub_chat_completion(chat_completion_body(content: "Acknowledged."))
+
+    raw_block = RubyLLM::Content::Raw.new([{ type: "text", text: "You are helpful" }])
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.add_message(role: :system, content: raw_block)
+    chat.add_message(role: :user, content: "Hi")
+    chat.complete
+
+    span = EXPORTER.finished_spans.first
+    system_instructions = JSON.parse(span.attributes["gen_ai.system_instructions"])
+    assert_equal(
+      [{ "type" => "raw", "content" => [{ "type" => "text", "text" => "You are helpful" }].to_json }],
+      system_instructions
+    )
+  ensure
+    OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = false
+  end
+
   def test_captures_content_when_enabled_via_env_var
     ENV["OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"] = "true"
 
-    stub_request(:post, "https://api.openai.com/v1/chat/completions")
-      .to_return(
-        status: 200,
-        headers: { "Content-Type" => "application/json" },
-        body: {
-          id: "chatcmpl-123",
-          object: "chat.completion",
-          model: "gpt-4o-mini",
-          choices: [{
-            index: 0,
-            message: { role: "assistant", content: "Hello, world!" },
-            finish_reason: "stop"
-          }],
-          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
-        }.to_json
-      )
+    stub_chat_completion
 
     chat = RubyLLM.chat(model: "gpt-4o-mini")
     chat.ask("Hi")

data/test/test_helper.rb

@@ -6,6 +6,36 @@ require "ruby_llm"
 require "opentelemetry/sdk"
 require "opentelemetry-instrumentation-ruby_llm"
 
+module ChatCompletionStubs
+  DEFAULT_USAGE = { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }.freeze
+
+  def chat_completion_body(content: "Hello, world!", model: "gpt-4o-mini", tool_calls: nil, usage: DEFAULT_USAGE)
+    message = { role: "assistant", content: content }
+    message[:tool_calls] = tool_calls if tool_calls
+
+    {
+      id: "chatcmpl-123",
+      object: "chat.completion",
+      model: model,
+      choices: [{
+        index: 0,
+        message: message,
+        finish_reason: tool_calls ? "tool_calls" : "stop"
+      }],
+      usage: usage
+    }.to_json
+  end
+
+  def stub_chat_completion(*bodies)
+    bodies = [chat_completion_body] if bodies.empty?
+    responses = bodies.map do |body|
+      { status: 200, headers: { "Content-Type" => "application/json" }, body: body }
+    end
+
+    stub_request(:post, "https://api.openai.com/v1/chat/completions").to_return(*responses)
+  end
+end
+
 EXPORTER = OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter.new
 span_processor = OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(EXPORTER)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: opentelemetry-instrumentation-ruby_llm
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Clarissa Borges
@@ -61,14 +61,20 @@ files:
 - example/trace_demonstration_with_langfuse_and_tools.rb
 - example/trace_demonstration_with_langfuse_ingredient_search.rb
 - example/trace_demonstration_with_tools.rb
+- gemfiles/ruby_llm_1.12.1.gemfile
 - gemfiles/ruby_llm_1.8.0.gemfile
 - gemfiles/ruby_llm_1_latest.gemfile
 - lib/opentelemetry-instrumentation-ruby_llm.rb
 - lib/opentelemetry/instrumentation/ruby_llm/instrumentation.rb
+- lib/opentelemetry/instrumentation/ruby_llm/message_formatter.rb
+- lib/opentelemetry/instrumentation/ruby_llm/patches/agent.rb
 - lib/opentelemetry/instrumentation/ruby_llm/patches/chat.rb
+- lib/opentelemetry/instrumentation/ruby_llm/patches/chat_methods.rb
 - lib/opentelemetry/instrumentation/ruby_llm/patches/embedding.rb
 - lib/opentelemetry/instrumentation/ruby_llm/version.rb
 - opentelemetry-instrumentation-ruby_llm.gemspec
+- test/active_record_chat_methods_test.rb
+- test/agent_instrumentation_test.rb
 - test/instrumentation_test.rb
 - test/test_helper.rb
 homepage: https://github.com/thoughtbot/opentelemetry-instrumentation-ruby_llm