opentelemetry-instrumentation-ruby_llm 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 07574aa081ca21455a0e4384ee35109310b70f878be921a6567b3126e9809602
-  data.tar.gz: 5a42c5a5f695c9b72cdf6a29caf5cd8d8ad8cf06895aebc0e170b9b4a02892c3
+  metadata.gz: c7dafd7bb3a8ccbb99ed902b457f203d0814a155c0604ea56ef68bdff17030a9
+  data.tar.gz: 21c19ccdf21c94db540dda4d557f3aa97406898f5dc17e4b3a18ee6d42ba9a69
 SHA512:
-  metadata.gz: 27d39b8ca21c044fc793c4e4d4b093b3b71d8010aecc52f518da5687943c7819f4eba144e1b4d10de889ab668344b0c952cbca5ac6f67e69e0110701b7f771ad
-  data.tar.gz: db4b50b51fb6f93a60cdee7738d9dbc181e3d18aec04fc0e6dda1a684b8e0c887be67b6bb1636c3f67580aa4decc3077b926128b40f45c216d8c4bd52ea99c7f
+  metadata.gz: ab14d77a71b7a964b98d92073dd3146f6b2c4917a393b25e524e67fdb101ef14b43a58b4e72384b1c77d5f08f9f21c3bf53e6ff71c2dcb30736fbc24fda2dc4d
+  data.tar.gz: 294d6307dbc72957bde11f5ef11b026d3575bce9794888fe788f73f3c95c06e7fbeb369feda88c5245db92974101e289c8ef0ed36c5263cb38895fd938c54317

data/.github/workflows/main.yml

@@ -9,14 +9,21 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-    name: Ruby ${{ matrix.ruby }}
+    name: Ruby ${{ matrix.ruby }} / ${{ matrix.gemfile }}
     strategy:
+      fail-fast: false
       matrix:
         ruby:
           - '3.4'
           - '3.3'
           - '3.2'
           - '3.1'
+        gemfile:
+          - gemfiles/ruby_llm_1.8.0.gemfile
+          - gemfiles/ruby_llm_1_latest.gemfile
+
+    env:
+      BUNDLE_GEMFILE: ${{ github.workspace }}/${{ matrix.gemfile }}
 
     steps:
       - uses: actions/checkout@v4

data/.gitignore

@@ -1,2 +1,3 @@
 *.gem
 Gemfile.lock
+gemfiles/*.gemfile.lock

data/Appraisals

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# Test the instrumentation against the earliest supported `ruby_llm` and
+# the latest 1.x release. 1.8.0 is the practical floor because the embedding
+# patch calls `RubyLLM::Models.resolve` (class method delegation added in
+# 1.8.0).
+
+appraise "ruby_llm-1.8.0" do
+  gem "ruby_llm", "1.8.0"
+end
+
+appraise "ruby_llm-1-latest" do
+  gem "ruby_llm", "~> 1.8"
+end

data/Gemfile

@@ -9,7 +9,8 @@ gem "opentelemetry-sdk"
 gem "opentelemetry-exporter-otlp"
 
 group :test do
+  gem "appraisal", "~> 2.5"
   gem "minitest"
-  gem "webmock"
   gem "rake"
+  gem "webmock"
 end

data/README.md

@@ -7,7 +7,7 @@ OpenTelemetry instrumentation for [RubyLLM](https://rubyllm.com).
 Install the gem using:
 
 ```sh
-gem opentelemetry-instrumentation-ruby_llm
+gem install opentelemetry-instrumentation-ruby_llm
 ```
 
 Or, if you use [bundler](https://bundler.io/), include `opentelemetry-instrumentation-ruby_llm` in your `Gemfile`.
@@ -96,11 +96,20 @@ Attributes persist across calls on the same chat instance and the method returns
 | Conversation tracking (`gen_ai.conversation.id`) | Planned |
 | System instructions capture | Supported (via `capture_content`) |
 | Custom attributes on traces and spans | Supported (via `with_otel_attributes`) |
-| Embeddings | Planned |
+| Embeddings | Supported |
 | Streaming | Planned |
 
 This gem follows the [OpenTelemetry GenAI Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/).
 
+## Compatibility
+
+This gem is tested against the following `ruby_llm` versions:
+
+- `1.8.0` (minimum supported)
+- `~> 1.8` (latest 1.x release)
+
+The Ruby matrix covers Ruby 3.1, 3.2, 3.3, and 3.4.
+
 ## License
 
 Copyright (c) Clarissa Borges and thoughtbot, inc.

data/gemfiles/ruby_llm_1.8.0.gemfile

@@ -0,0 +1,16 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "ruby_llm", "1.8.0"
+gem "opentelemetry-sdk"
+gem "opentelemetry-exporter-otlp"
+
+group :test do
+  gem "appraisal", "~> 2.5"
+  gem "minitest"
+  gem "rake"
+  gem "webmock"
+end
+
+gemspec path: "../"

data/gemfiles/ruby_llm_1_latest.gemfile

@@ -0,0 +1,16 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "ruby_llm", "~> 1.8"
+gem "opentelemetry-sdk"
+gem "opentelemetry-exporter-otlp"
+
+group :test do
+  gem "appraisal", "~> 2.5"
+  gem "minitest"
+  gem "rake"
+  gem "webmock"
+end
+
+gemspec path: "../"

data/lib/opentelemetry/instrumentation/ruby_llm/instrumentation.rb

@@ -4,6 +4,8 @@ module OpenTelemetry
   module Instrumentation
     module RubyLLM
       class Instrumentation < OpenTelemetry::Instrumentation::Base
+        MINIMUM_RUBY_LLM_VERSION = "1.8.0"
+
         instrumentation_name "OpenTelemetry::Instrumentation::RubyLLM"
         instrumentation_version VERSION
 
@@ -13,6 +15,22 @@ module OpenTelemetry
           defined?(::RubyLLM)
         end
 
+        compatible do
+          # The embedding patch calls `RubyLLM::Models.resolve` (class-method delegation added in 1.8.0);
+          # Anything older than 1.8.0 would NoMethodError / NameError at install or first use.
+          compatible = Gem::Version.new(::RubyLLM::VERSION) >= Gem::Version.new(MINIMUM_RUBY_LLM_VERSION)
+
+          unless compatible
+            OpenTelemetry.logger.warn(
+              "[OpenTelemetry::Instrumentation::RubyLLM] ruby_llm " \
+              "#{::RubyLLM::VERSION} is below the required minimum " \
+              "#{MINIMUM_RUBY_LLM_VERSION}; instrumentation will not be installed."
+            )
+          end
+
+          compatible
+        end
+
         install do |_config|
           require_relative "patches/chat"
           require_relative "patches/embedding"

data/lib/opentelemetry/instrumentation/ruby_llm/patches/chat.rb

@@ -19,6 +19,9 @@ module OpenTelemetry
               "gen_ai.provider.name" => provider,
               "gen_ai.request.model" => model_id,
             }
+            # Per GenAI semconv: set `gen_ai.request.stream` if and only if
+            # the request is streaming. Absence means non-streaming.
+            attributes["gen_ai.request.stream"] = true if block_given?
 
             tracer.in_span("chat #{model_id}", attributes: attributes, kind: OpenTelemetry::Trace::SpanKind::CLIENT) do |span|
               begin
@@ -37,6 +40,16 @@ module OpenTelemetry
                 span.set_attribute("gen_ai.usage.output_tokens", response.output_tokens) if response.output_tokens
                 span.set_attribute("gen_ai.request.temperature", @temperature) if @temperature
 
+                # Prompt-cache token accessors were added in ruby_llm 1.9.0 (commit 869a755f).
+                if response.respond_to?(:cached_tokens) && response.cached_tokens
+                  span.set_attribute("gen_ai.usage.cache_read.input_tokens", response.cached_tokens)
+                end
+
+                # Prompt-cache token accessors were added in ruby_llm 1.9.0 (commit 869a755f).
+                if response.respond_to?(:cache_creation_tokens) && response.cache_creation_tokens
+                  span.set_attribute("gen_ai.usage.cache_creation.input_tokens", response.cache_creation_tokens)
+                end
+
                 if capture_content?
                   system_messages = @messages.select { |m| m.role == :system }
                   input_messages = @messages[0..-2].reject { |m| m.role == :system }
@@ -58,16 +71,28 @@ module OpenTelemetry
 
           def execute_tool(tool_call)
             attributes = {
+              "gen_ai.operation.name" => "execute_tool",
               "gen_ai.tool.name" => tool_call.name,
               "gen_ai.tool.call.id" => tool_call.id,
               "gen_ai.tool.call.arguments" => tool_call.arguments.to_json,
-              "gen_ai.tool.type" => "function"
-            }
+              "gen_ai.tool.type" => "function",
+              "gen_ai.tool.description" => tools[tool_call.name.to_sym]&.description
+            }.compact
 
             tracer.in_span("execute_tool #{tool_call.name}", attributes: attributes, kind: OpenTelemetry::Trace::SpanKind::INTERNAL) do |span|
-              result = super
-              result_str = result.is_a?(::RubyLLM::Tool::Halt) ? result.content.to_s : result.to_s
-              span.set_attribute("gen_ai.tool.call.result", result_str[0..500])
+              begin
+                result = super
+              rescue => e
+                span.record_exception(e)
+                span.status = OpenTelemetry::Trace::Status.error(e.message)
+                span.set_attribute("error.type", e.class.name)
+                raise
+              end
+
+              # `RubyLLM::Tool::Halt#to_s` returns `@content.to_s`, so a single
+              # `to_s` covers both the Halt and plain-result cases.
+              span.set_attribute("gen_ai.tool.call.result", result.to_s[0..500])
+
               result
             end
           end

data/lib/opentelemetry/instrumentation/ruby_llm/version.rb

@@ -3,7 +3,7 @@
 module OpenTelemetry
   module Instrumentation
     module RubyLLM
-      VERSION = "0.5.0"
+      VERSION = "0.6.0"
     end
   end
 end

data/test/instrumentation_test.rb

@@ -9,6 +9,27 @@ class InstrumentationTest < Minitest::Test
     end
   end
 
+  def test_compatible_is_true_for_current_ruby_llm_version
+    instrumentation = OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance
+    assert_equal true, instrumentation.compatible?
+  end
+
+  def test_compatible_is_false_when_ruby_llm_below_minimum
+    original_version = ::RubyLLM::VERSION
+    ::RubyLLM.send(:remove_const, :VERSION)
+    ::RubyLLM.const_set(:VERSION, "1.7.99")
+
+    instrumentation = OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance
+    assert_equal false, instrumentation.compatible?
+  ensure
+    ::RubyLLM.send(:remove_const, :VERSION)
+    ::RubyLLM.const_set(:VERSION, original_version)
+  end
+
+  def test_minimum_ruby_llm_version_is_pinned_at_1_8_0
+    assert_equal "1.8.0", OpenTelemetry::Instrumentation::RubyLLM::Instrumentation::MINIMUM_RUBY_LLM_VERSION
+  end
+
   def test_creates_span_with_attributes
     stub_request(:post, "https://api.openai.com/v1/chat/completions")
       .to_return(
@@ -45,10 +66,61 @@ class InstrumentationTest < Minitest::Test
     assert_equal "openai", span.attributes["gen_ai.provider.name"]
     assert_equal "gpt-4o-mini", span.attributes["gen_ai.request.model"]
     assert_equal "chat", span.attributes["gen_ai.operation.name"]
+    # Per GenAI semconv, `gen_ai.request.stream` is set only when streaming.
+    assert_nil span.attributes["gen_ai.request.stream"]
     assert_equal 10, span.attributes["gen_ai.usage.input_tokens"]
     assert_equal 5, span.attributes["gen_ai.usage.output_tokens"]
   end
 
+  def test_marks_streaming_chat_requests
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        status: 200,
+        headers: { "Content-Type" => "application/json" },
+        body: {
+          id: "chatcmpl-123",
+          model: "gpt-4o-mini",
+          choices: [{ index: 0, message: { role: "assistant", content: "Hi" }, finish_reason: "stop" }],
+          usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }
+        }.to_json
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.ask("Hi") { |_chunk| }
+
+    span = EXPORTER.finished_spans.first
+    assert_equal true, span.attributes["gen_ai.request.stream"]
+  end
+
+  def test_records_prompt_cache_tokens
+    # RubyLLM's OpenAI provider maps `cached_tokens` ← `cache_read_tokens(usage)`
+    # and `cache_creation_tokens` ← `cache_write_tokens(usage)`, both surfaced
+    # on `Message#cached_tokens` / `Message#cache_creation_tokens`.
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        status: 200,
+        headers: { "Content-Type" => "application/json" },
+        body: {
+          id: "chatcmpl-cache",
+          model: "gpt-4o-mini",
+          choices: [{ index: 0, message: { role: "assistant", content: "Hello!" }, finish_reason: "stop" }],
+          usage: {
+            prompt_tokens: 100,
+            completion_tokens: 5,
+            total_tokens: 105,
+            prompt_tokens_details: { cached_tokens: 75, cache_write_tokens: 20 }
+          }
+        }.to_json
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.ask("Hi")
+
+    span = EXPORTER.finished_spans.first
+    assert_equal 75, span.attributes["gen_ai.usage.cache_read.input_tokens"]
+    assert_equal 20, span.attributes["gen_ai.usage.cache_creation.input_tokens"]
+  end
+
   def test_records_error_on_api_failure
     stub_request(:post, "https://api.openai.com/v1/chat/completions")
       .to_return(status: 500, body: "Internal Server Error")
@@ -168,13 +240,57 @@ class InstrumentationTest < Minitest::Test
     tool_span = tool_spans.first
     assert_equal OpenTelemetry::Trace::SpanKind::INTERNAL, tool_span.kind
     assert_equal "execute_tool calculator", tool_span.name
+    assert_equal "execute_tool", tool_span.attributes["gen_ai.operation.name"]
     assert_equal "calculator", tool_span.attributes["gen_ai.tool.name"]
+    assert_equal "Performs math", tool_span.attributes["gen_ai.tool.description"]
     assert_equal '{"expression":"2+2"}', tool_span.attributes["gen_ai.tool.call.arguments"]
     assert_equal "4", tool_span.attributes["gen_ai.tool.call.result"]
     assert_equal "call_abc123", tool_span.attributes["gen_ai.tool.call.id"]
     assert_equal "function", tool_span.attributes["gen_ai.tool.type"]
   end
 
+  def test_records_error_when_tool_raises
+    boom = Class.new(RubyLLM::Tool) do
+      def self.name = "boom"
+      description "Always raises"
+
+      def execute
+        raise ArgumentError, "tool failure"
+      end
+    end
+
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        status: 200,
+        headers: { "Content-Type" => "application/json" },
+        body: {
+          id: "chatcmpl-boom",
+          model: "gpt-4o-mini",
+          choices: [{
+            index: 0,
+            message: {
+              role: "assistant",
+              content: nil,
+              tool_calls: [{
+                id: "call_x",
+                type: "function",
+                function: { name: "boom", arguments: "{}" }
+              }]
+            },
+            finish_reason: "tool_calls"
+          }],
+          usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }
+        }.to_json
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini").with_tool(boom)
+    assert_raises(ArgumentError) { chat.ask("trigger") }
+
+    tool_span = EXPORTER.finished_spans.find { |s| s.name.start_with?("execute_tool ") }
+    assert_equal "ArgumentError", tool_span.attributes["error.type"]
+    assert_equal OpenTelemetry::Trace::Status::ERROR, tool_span.status.code
+  end
+
   def test_does_not_capture_content_by_default
     stub_request(:post, "https://api.openai.com/v1/chat/completions")
       .to_return(

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: opentelemetry-instrumentation-ruby_llm
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Clarissa Borges
@@ -48,6 +48,7 @@ files:
 - ".github/workflows/dynamic-security.yml"
 - ".github/workflows/main.yml"
 - ".gitignore"
+- Appraisals
 - CODEOWNERS
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -60,6 +61,8 @@ files:
 - example/trace_demonstration_with_langfuse_and_tools.rb
 - example/trace_demonstration_with_langfuse_ingredient_search.rb
 - example/trace_demonstration_with_tools.rb
+- gemfiles/ruby_llm_1.8.0.gemfile
+- gemfiles/ruby_llm_1_latest.gemfile
 - lib/opentelemetry-instrumentation-ruby_llm.rb
 - lib/opentelemetry/instrumentation/ruby_llm/instrumentation.rb
 - lib/opentelemetry/instrumentation/ruby_llm/patches/chat.rb