opentelemetry-instrumentation-ruby_llm 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e43dc7d2e1f91e06c28aa8237a6393d565605f92635edf69e2948015e5d5c4b
-  data.tar.gz: b7eb8e02952582e6d8af04f8ee04407ac5e880b9f992ccdd4e6e353e3c060af0
+  metadata.gz: 25a2701b37bc9acecc70d4ff326230cbf5e073454f47c453744ced08d468b2f8
+  data.tar.gz: a5885383024ecf52deb132bdf0ccb86b1cb9a0169561a714e6d3cabec8e3a64a
 SHA512:
-  metadata.gz: b600d97f321e3626ffa9de8785e67762d3f391b60e5d11a5d3f97657cbd4801e8cb7d1dd86e632bc260873732772176371b75a1fe17f6325fc2815fb23dc4b9b
-  data.tar.gz: 76384730ef9cfe8cfe7f1d5b68109c6cbd4bedb897f18393dc717e0e5089d3ec7cbd81f21f728db4e82bc73ff72f08765b75b97e980ef665d5662ad5321759c5
+  metadata.gz: 023c166358bee510b9cd75f41764179e019eba7f3df1153c8d3806a8ce7488b7781688b708a2b315b4a2207306d822cf063c2e2125874cbbe3fc912176897edd
+  data.tar.gz: ef4d632a49a4d22d66d751648a2c5edf6ba13d0223dc1d42901193f675e13eff065ea8dbf713bac7f29450b2af6fd510126e4bbf5a713138bf16b9a74152c642

data/.github/workflows/main.yml

@@ -16,6 +16,7 @@ jobs:
           - '3.4'
           - '3.3'
           - '3.2'
+          - '3.1'
 
     steps:
       - uses: actions/checkout@v4

data/lib/opentelemetry/instrumentation/ruby_llm/instrumentation.rb

@@ -15,7 +15,9 @@ module OpenTelemetry
 
         install do |_config|
           require_relative "patches/chat"
+          require_relative "patches/embedding"
           ::RubyLLM::Chat.prepend(Patches::Chat)
+          ::RubyLLM::Embedding.singleton_class.prepend(Patches::Embedding)
         end
       end
     end

data/lib/opentelemetry/instrumentation/ruby_llm/patches/chat.rb

@@ -5,7 +5,7 @@ module OpenTelemetry
     module RubyLLM
       module Patches
         module Chat
-          def ask(message, &block)
+          def complete(&)
             provider = @model&.provider || "unknown"
             model_id = @model&.id || "unknown"
 
@@ -47,9 +47,6 @@ module OpenTelemetry
 
               result
             end
-          rescue StandardError => e
-            OpenTelemetry.handle_error(exception: e)
-            super
           end
 
           def execute_tool(tool_call)
@@ -66,9 +63,6 @@ module OpenTelemetry
               span.set_attribute("gen_ai.tool.call.result", result_str[0..500])
               result
             end
-          rescue StandardError => e
-            OpenTelemetry.handle_error(exception: e)
-            super
           end
 
           private

data/lib/opentelemetry/instrumentation/ruby_llm/patches/embedding.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module OpenTelemetry
+  module Instrumentation
+    module RubyLLM
+      module Patches
+        module Embedding
+          def embed(text, model: nil, provider: nil, assume_model_exists: false, context: nil, dimensions: nil)
+            config = context&.config || ::RubyLLM.config
+            resolved_model = model || config.default_embedding_model
+            model_obj, _provider_instance = ::RubyLLM::Models.resolve(
+              resolved_model, provider: provider, assume_exists: assume_model_exists, config: config
+            )
+            model_id = model_obj.id
+            provider_name = model_obj.provider || "unknown"
+
+            attributes = {
+              "gen_ai.operation.name" => "embeddings",
+              "gen_ai.provider.name" => provider_name,
+              "gen_ai.request.model" => model_id
+            }
+
+            tracer.in_span("embeddings #{model_id}", attributes: attributes, kind: OpenTelemetry::Trace::SpanKind::CLIENT) do |span|
+              begin
+                result = super
+              rescue => e
+                span.record_exception(e)
+                span.status = OpenTelemetry::Trace::Status.error(e.message)
+                span.set_attribute("error.type", e.class.name)
+                raise
+              end
+
+              span.set_attribute("gen_ai.response.model", result.model) if result.model
+              span.set_attribute("gen_ai.usage.input_tokens", result.input_tokens) if result.input_tokens&.positive?
+
+              if result.vectors.is_a?(Array)
+                first = result.vectors.first
+                vector = first.is_a?(Array) ? first : result.vectors
+                span.set_attribute("gen_ai.embeddings.dimension.count", vector.length) if vector.is_a?(Array)
+              end
+
+              result
+            end
+          end
+
+          private
+
+          def tracer
+            RubyLLM::Instrumentation.instance.tracer
+          end
+        end
+      end
+    end
+  end
+end

data/lib/opentelemetry/instrumentation/ruby_llm/version.rb

@@ -3,7 +3,7 @@
 module OpenTelemetry
   module Instrumentation
     module RubyLLM
-      VERSION = "0.2.0"
+      VERSION = "0.4.0"
     end
   end
 end

data/opentelemetry-instrumentation-ruby_llm.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
   spec.description = "Adds OpenTelemetry tracing to RubyLLM chat operations"
   spec.homepage    = "https://github.com/thoughtbot/opentelemetry-instrumentation-ruby_llm"
 
-  spec.required_ruby_version = ">= 3.2.0"
+  spec.required_ruby_version = ">= 3.1.3"
 
   spec.metadata["homepage_uri"] = spec.homepage
 

data/test/instrumentation_test.rb

@@ -67,7 +67,7 @@ class InstrumentationTest < Minitest::Test
     assert_equal OpenTelemetry::Trace::Status::ERROR, span.status.code
   end
 
-  def test_ask_still_works_when_instrumentation_fails
+  def test_instruments_complete_called_directly
     stub_request(:post, "https://api.openai.com/v1/chat/completions")
       .to_return(
         status: 200,
@@ -78,7 +78,7 @@ class InstrumentationTest < Minitest::Test
           model: "gpt-4o-mini",
           choices: [{
             index: 0,
-            message: { role: "assistant", content: "Hello!" },
+            message: { role: "assistant", content: "Hello, world!" },
             finish_reason: "stop"
           }],
           usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
@@ -86,10 +86,18 @@ class InstrumentationTest < Minitest::Test
       )
 
     chat = RubyLLM.chat(model: "gpt-4o-mini")
-    chat.define_singleton_method(:tracer) { raise StandardError, "instrumentation bug" }
+    chat.add_message(role: :user, content: "Hi")
+    chat.complete
+
+    spans = EXPORTER.finished_spans
+    assert_equal 1, spans.length
 
-    response = chat.ask("Hi")
-    assert_equal "Hello!", response.content
+    span = spans.first
+    assert_equal "chat gpt-4o-mini", span.name
+    assert_equal "chat", span.attributes["gen_ai.operation.name"]
+    assert_equal "openai", span.attributes["gen_ai.provider.name"]
+    assert_equal 10, span.attributes["gen_ai.usage.input_tokens"]
+    assert_equal 5, span.attributes["gen_ai.usage.output_tokens"]
   end
 
   def test_creates_span_for_tool_call
@@ -155,7 +163,7 @@ class InstrumentationTest < Minitest::Test
     chat_spans = spans.select { |s| s.name.include?("chat ") }
 
     assert_equal 1, tool_spans.length
-    assert_equal 1, chat_spans.length
+    assert_equal 2, chat_spans.length
 
     tool_span = tool_spans.first
     assert_equal OpenTelemetry::Trace::SpanKind::INTERNAL, tool_span.kind
@@ -167,68 +175,6 @@ class InstrumentationTest < Minitest::Test
     assert_equal "function", tool_span.attributes["gen_ai.tool.type"]
   end
 
-  def test_execute_tool_still_works_when_instrumentation_fails
-    calculator = Class.new(RubyLLM::Tool) do
-      def self.name = "calculator"
-      description "Performs math"
-      param :expression, type: "string", desc: "Math expression"
-
-      def execute(expression:)
-        eval(expression).to_s
-      end
-    end
-
-    stub_request(:post, "https://api.openai.com/v1/chat/completions")
-      .to_return(
-        {
-          status: 200,
-          headers: { "Content-Type" => "application/json" },
-          body: {
-            id: "chatcmpl-123",
-            object: "chat.completion",
-            model: "gpt-4o-mini",
-            choices: [{
-              index: 0,
-              message: {
-                role: "assistant",
-                content: nil,
-                tool_calls: [{
-                  id: "call_abc123",
-                  type: "function",
-                  function: { name: "calculator", arguments: '{"expression":"2+2"}' }
-                }]
-              },
-              finish_reason: "tool_calls"
-            }],
-            usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
-          }.to_json
-        },
-        {
-          status: 200,
-          headers: { "Content-Type" => "application/json" },
-          body: {
-            id: "chatcmpl-456",
-            object: "chat.completion",
-            model: "gpt-4o-mini",
-            choices: [{
-              index: 0,
-              message: { role: "assistant", content: "The answer is 4" },
-              finish_reason: "stop"
-            }],
-            usage: { prompt_tokens: 20, completion_tokens: 5, total_tokens: 25 }
-          }.to_json
-        }
-      )
-
-    chat = RubyLLM.chat(model: "gpt-4o-mini")
-    chat.with_tool(calculator)
-
-    chat.define_singleton_method(:tracer) { raise StandardError, "instrumentation bug" }
-
-    response = chat.ask("What is 2+2?")
-    assert_equal "The answer is 4", response.content
-  end
-
   def test_does_not_capture_content_by_default
     stub_request(:post, "https://api.openai.com/v1/chat/completions")
       .to_return(
@@ -299,6 +245,53 @@ class InstrumentationTest < Minitest::Test
     OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = false
   end
 
+  def test_creates_span_for_embedding
+    stub_request(:post, "https://api.openai.com/v1/embeddings")
+      .to_return(
+        status: 200,
+        headers: { "Content-Type" => "application/json" },
+        body: {
+          object: "list",
+          model: "text-embedding-3-small",
+          data: [
+            { object: "embedding", index: 0, embedding: [0.1, 0.2, 0.3] }
+          ],
+          usage: { prompt_tokens: 8, total_tokens: 8 }
+        }.to_json
+      )
+
+    RubyLLM.embed("Hello, world!", model: "text-embedding-3-small")
+
+    spans = EXPORTER.finished_spans
+    assert_equal 1, spans.length
+
+    span = spans.first
+    assert_equal OpenTelemetry::Trace::SpanKind::CLIENT, span.kind
+    assert_equal "embeddings text-embedding-3-small", span.name
+    assert_equal "embeddings", span.attributes["gen_ai.operation.name"]
+    assert_equal "openai", span.attributes["gen_ai.provider.name"]
+    assert_equal "text-embedding-3-small", span.attributes["gen_ai.request.model"]
+    assert_equal "text-embedding-3-small", span.attributes["gen_ai.response.model"]
+    assert_equal 8, span.attributes["gen_ai.usage.input_tokens"]
+    assert_equal 3, span.attributes["gen_ai.embeddings.dimension.count"]
+  end
+
+  def test_records_error_on_embedding_api_failure
+    stub_request(:post, "https://api.openai.com/v1/embeddings")
+      .to_return(status: 500, body: "Internal Server Error")
+
+    assert_raises do
+      RubyLLM.embed("Hello", model: "text-embedding-3-small")
+    end
+
+    spans = EXPORTER.finished_spans
+    span = spans.last
+
+    assert_equal "embeddings text-embedding-3-small", span.name
+    assert span.attributes["error.type"]
+    assert_equal OpenTelemetry::Trace::Status::ERROR, span.status.code
+  end
+
   def test_captures_content_when_enabled_via_env_var
     ENV["OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"] = "true"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: opentelemetry-instrumentation-ruby_llm
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Clarissa Borges
@@ -62,6 +62,7 @@ files:
 - lib/opentelemetry-instrumentation-ruby_llm.rb
 - lib/opentelemetry/instrumentation/ruby_llm/instrumentation.rb
 - lib/opentelemetry/instrumentation/ruby_llm/patches/chat.rb
+- lib/opentelemetry/instrumentation/ruby_llm/patches/embedding.rb
 - lib/opentelemetry/instrumentation/ruby_llm/version.rb
 - opentelemetry-instrumentation-ruby_llm.gemspec
 - test/instrumentation_test.rb
@@ -78,7 +79,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.2.0
+      version: 3.1.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="