opentelemetry-instrumentation-ruby_llm 0.3.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dfb17d4d74d31d2a53792683e0d5d070cd25e9160ee3c5018c0a1dbfb5bbeb4b
-  data.tar.gz: 91c3b7ac277d43ddf80a6fc49e2228d8005bf40f9321cd10cd5ac2539c96dfc7
+  metadata.gz: 07574aa081ca21455a0e4384ee35109310b70f878be921a6567b3126e9809602
+  data.tar.gz: 5a42c5a5f695c9b72cdf6a29caf5cd8d8ad8cf06895aebc0e170b9b4a02892c3
 SHA512:
-  metadata.gz: df479838c253aa695c1b36b393b4f7eac31dd041a581df3d04d9f9e9ca1253d026b8582b32d2cdf398e812d53459ef0c012406654342d37a9ae45c45dc68c5e7
-  data.tar.gz: 9c18a05253193ce23af8bc0f342cd726778c6daccff7d1010602a63b36f20d07eb13852ea6f01d6e40bf2439236fbf4b495e32e144e701d521aaeec18972f6a4
+  metadata.gz: 27d39b8ca21c044fc793c4e4d4b093b3b71d8010aecc52f518da5687943c7819f4eba144e1b4d10de889ab668344b0c952cbca5ac6f67e69e0110701b7f771ad
+  data.tar.gz: db4b50b51fb6f93a60cdee7738d9dbc181e3d18aec04fc0e6dda1a684b8e0c887be67b6bb1636c3f67580aa4decc3077b926128b40f45c216d8c4bd52ea99c7f

data/.github/workflows/main.yml

@@ -16,6 +16,7 @@ jobs:
           - '3.4'
           - '3.3'
           - '3.2'
+          - '3.1'
 
     steps:
       - uses: actions/checkout@v4

data/README.md

@@ -59,6 +59,32 @@ When enabled, the following attributes are added to chat spans:
 > [!WARNING]
 > Captured content may include sensitive or personally identifiable information (PII). Use with caution in production environments.
 
+### Custom attributes
+
+Use `with_otel_attributes` to add arbitrary attributes to the span for each request. This is useful for adding per-request metadata like Langfuse prompt linking or trace-level tags:
+
+```ruby
+chat = RubyLLM.chat
+chat.with_otel_attributes(
+  "langfuse.observation.prompt.name" => "supplement-assistant",
+  "langfuse.observation.prompt.version" => 1,
+  "langfuse.trace.tags" => ["vitamins"],
+  "langfuse.trace.metadata" => { category: "health" }.to_json
+)
+chat.ask("What are the side effects of Vitamin D3?")
+```
+
+Values can also be callables (Procs/lambdas) that are evaluated after each completion, giving access to response data:
+
+```ruby
+chat.with_otel_attributes(
+  "langfuse.observation.prompt.name" => "supplement-assistant",
+  "langfuse.observation.output" => -> { chat.messages.last&.content.to_s }
+)
+```
+
+Attributes persist across calls on the same chat instance and the method returns `self` for chaining.
+
 ## What's traced?
 
 | Feature | Status |
@@ -68,8 +94,8 @@ When enabled, the following attributes are added to chat spans:
 | Error handling | Supported |
 | Opt-in input/output content capture | Supported |
 | Conversation tracking (`gen_ai.conversation.id`) | Planned |
-| System instructions capture | Planned |
-| Custom attributes on traces and spans | Planned |
+| System instructions capture | Supported (via `capture_content`) |
+| Custom attributes on traces and spans | Supported (via `with_otel_attributes`) |
 | Embeddings | Planned |
 | Streaming | Planned |
 

data/example/trace_demonstration_with_langfuse.rb

@@ -36,6 +36,10 @@ end
 
 chat = RubyLLM.chat
 chat.with_instructions("You are a helpful assistant that provides concise answers.")
+chat.with_otel_attributes(
+  "langfuse.observation.prompt.name" => "helpful-assistant",
+  "langfuse.observation.prompt.version" => 1
+)
 response = chat.ask("What is the meaning of life?")
 puts "\nResponse: #{response.content}"
 

data/example/trace_demonstration_with_langfuse_and_tools.rb

@@ -46,6 +46,10 @@ end
 chat = RubyLLM.chat
 chat.with_instructions("You are a helpful assistant that provides concise answers.")
 chat.with_tool(Calculator)
+chat.with_otel_attributes(
+  "langfuse.observation.prompt.name" => "helpful-assistant",
+  "langfuse.observation.prompt.version" => 1
+)
 response = chat.ask("Use the calculator tool to compute 123 * 456")
 puts "\nResponse: #{response.content}"
 response = chat.ask("Use the tool again to compute 789 + 1011")

data/example/trace_demonstration_with_langfuse_ingredient_search.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require "bundler/inline"
+
+gemfile(true) do
+  source "https://rubygems.org"
+  gem "ruby_llm"
+  gem "opentelemetry-api"
+  gem "opentelemetry-sdk"
+  gem "opentelemetry-exporter-otlp"
+  gem "opentelemetry-instrumentation-ruby_llm", path: "../"
+  gem "base64"
+  gem "dotenv"
+end
+
+require "base64"
+require "dotenv/load"
+
+credentials = Base64.strict_encode64("#{ENV['LANGFUSE_PUBLIC_KEY']}:#{ENV['LANGFUSE_SECRET_KEY']}")
+
+OpenTelemetry::SDK.configure do |c|
+  c.service_name = "ruby_llm-demo"
+  c.add_span_processor(
+    OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
+      OpenTelemetry::Exporter::OTLP::Exporter.new(
+        endpoint: "https://us.cloud.langfuse.com/api/public/otel/v1/traces",
+        headers: { "Authorization" => "Basic #{credentials}" }
+      )
+    )
+  )
+  c.use "OpenTelemetry::Instrumentation::RubyLLM", capture_content: true
+end
+
+RubyLLM.configure do |c|
+  c.openai_api_key = ENV["OPENAI_API_KEY"]
+  c.default_model = "gpt-5-nano"
+end
+
+INGREDIENT_DATABASE = {
+  "vitamin d3" => {
+    name: "Vitamin D3 (Cholecalciferol)",
+    common_doses: "1,000-5,000 IU daily",
+    side_effects: ["Nausea", "Vomiting", "Constipation", "Loss of appetite", "Excessive thirst", "Frequent urination", "Kidney stones (at very high doses)"],
+    interactions: ["Corticosteroids", "Orlistat", "Statins", "Thiazide diuretics"],
+    notes: "Fat-soluble vitamin. Toxicity risk at sustained doses above 10,000 IU/day."
+  },
+  "magnesium glycinate" => {
+    name: "Magnesium Glycinate",
+    common_doses: "200-400 mg daily",
+    side_effects: ["Diarrhea", "Nausea", "Abdominal cramping"],
+    interactions: ["Antibiotics (tetracyclines, quinolones)", "Bisphosphonates", "Diuretics"],
+    notes: "Better absorbed and gentler on the stomach than magnesium oxide."
+  },
+  "zinc" => {
+    name: "Zinc",
+    common_doses: "15-30 mg daily",
+    side_effects: ["Nausea", "Metallic taste", "Headache", "Copper deficiency (long-term use)"],
+    interactions: ["Antibiotics", "Penicillamine", "Copper supplements"],
+    notes: "Best taken with food to reduce nausea. Long-term use above 40 mg/day may deplete copper."
+  }
+}
+
+class SearchForIngredientDetails < RubyLLM::Tool
+  description "Searches a database for detailed information about a supplement ingredient, including side effects, interactions, and dosage"
+  param :ingredient_name, type: "string", desc: "The name of the ingredient to search for (e.g., 'vitamin d3', 'magnesium glycinate')"
+
+  def execute(ingredient_name:)
+    key = ingredient_name.downcase.strip
+    match = INGREDIENT_DATABASE.find { |k, _| key.include?(k) || k.include?(key) }
+
+    if match
+      _, details = match
+      details.map { |k, v| "#{k}: #{Array(v).join(', ')}" }.join("\n")
+    else
+      "No information found for '#{ingredient_name}'. Available ingredients: #{INGREDIENT_DATABASE.keys.join(', ')}"
+    end
+  end
+end
+
+chat = RubyLLM.chat
+chat.with_instructions("You are a knowledgeable health supplement assistant. Use the search tool to look up ingredient details before answering questions.")
+chat.with_tool(SearchForIngredientDetails)
+
+questions = [
+  { text: "What are the side effects of Vitamin D3?", ingredient: "vitamin d3" },
+  { text: "What are the common interactions with magnesium glycinate?", ingredient: "magnesium glycinate" },
+  { text: "What is the recommended dosage for zinc?", ingredient: "zinc" },
+  { text: "Are there any interactions I should be aware of with zinc?", ingredient: "zinc" }
+]
+
+questions.each do |q|
+  puts "\n---\n\n"
+  puts "Question: #{q[:text]}\n\n"
+
+  chat.with_otel_attributes(
+    "langfuse.observation.prompt.name" => "supplement-assistant",
+    "langfuse.observation.prompt.version" => 1,
+    "langfuse.observation.input" => q[:text],
+    "langfuse.observation.output" => -> { chat.messages.last&.content.to_s },
+    "langfuse.observation.metadata" => { ingredient: q[:ingredient] }.to_json,
+    "langfuse.trace.metadata" => { ingredient: q[:ingredient] }.to_json,
+    "langfuse.trace.tags" => [q[:ingredient]]
+  )
+
+  response = chat.ask(q[:text])
+  puts "\nResponse: #{response.content}"
+end
+
+# This line is only necessary in short-lived scripts. In a long-running application, spans will be flushed automatically.
+OpenTelemetry.tracer_provider.force_flush

data/lib/opentelemetry/instrumentation/ruby_llm/instrumentation.rb

@@ -15,7 +15,9 @@ module OpenTelemetry
 
         install do |_config|
           require_relative "patches/chat"
+          require_relative "patches/embedding"
           ::RubyLLM::Chat.prepend(Patches::Chat)
+          ::RubyLLM::Embedding.singleton_class.prepend(Patches::Embedding)
         end
       end
     end

data/lib/opentelemetry/instrumentation/ruby_llm/patches/chat.rb

@@ -5,6 +5,11 @@ module OpenTelemetry
     module RubyLLM
       module Patches
         module Chat
+          def with_otel_attributes(attributes)
+            @otel_attributes = attributes
+            self
+          end
+
           def complete(&)
             provider = @model&.provider || "unknown"
             model_id = @model&.id || "unknown"
@@ -45,11 +50,10 @@ module OpenTelemetry
                 end
               end
 
+              @otel_attributes&.each { |key, value| span.set_attribute(key, value.respond_to?(:call) ? value.call : value) }
+
               result
             end
-          rescue StandardError => e
-            OpenTelemetry.handle_error(exception: e)
-            super
           end
 
           def execute_tool(tool_call)
@@ -66,9 +70,6 @@ module OpenTelemetry
               span.set_attribute("gen_ai.tool.call.result", result_str[0..500])
               result
             end
-          rescue StandardError => e
-            OpenTelemetry.handle_error(exception: e)
-            super
           end
 
           private

data/lib/opentelemetry/instrumentation/ruby_llm/patches/embedding.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module OpenTelemetry
+  module Instrumentation
+    module RubyLLM
+      module Patches
+        module Embedding
+          def embed(text, model: nil, provider: nil, assume_model_exists: false, context: nil, dimensions: nil)
+            config = context&.config || ::RubyLLM.config
+            resolved_model = model || config.default_embedding_model
+            model_obj, _provider_instance = ::RubyLLM::Models.resolve(
+              resolved_model, provider: provider, assume_exists: assume_model_exists, config: config
+            )
+            model_id = model_obj.id
+            provider_name = model_obj.provider || "unknown"
+
+            attributes = {
+              "gen_ai.operation.name" => "embeddings",
+              "gen_ai.provider.name" => provider_name,
+              "gen_ai.request.model" => model_id
+            }
+
+            tracer.in_span("embeddings #{model_id}", attributes: attributes, kind: OpenTelemetry::Trace::SpanKind::CLIENT) do |span|
+              begin
+                result = super
+              rescue => e
+                span.record_exception(e)
+                span.status = OpenTelemetry::Trace::Status.error(e.message)
+                span.set_attribute("error.type", e.class.name)
+                raise
+              end
+
+              span.set_attribute("gen_ai.response.model", result.model) if result.model
+              span.set_attribute("gen_ai.usage.input_tokens", result.input_tokens) if result.input_tokens&.positive?
+
+              if result.vectors.is_a?(Array)
+                first = result.vectors.first
+                vector = first.is_a?(Array) ? first : result.vectors
+                span.set_attribute("gen_ai.embeddings.dimension.count", vector.length) if vector.is_a?(Array)
+              end
+
+              result
+            end
+          end
+
+          private
+
+          def tracer
+            RubyLLM::Instrumentation.instance.tracer
+          end
+        end
+      end
+    end
+  end
+end

data/lib/opentelemetry/instrumentation/ruby_llm/version.rb

@@ -3,7 +3,7 @@
 module OpenTelemetry
   module Instrumentation
     module RubyLLM
-      VERSION = "0.3.0"
+      VERSION = "0.5.0"
     end
   end
 end

data/opentelemetry-instrumentation-ruby_llm.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
   spec.description = "Adds OpenTelemetry tracing to RubyLLM chat operations"
   spec.homepage    = "https://github.com/thoughtbot/opentelemetry-instrumentation-ruby_llm"
 
-  spec.required_ruby_version = ">= 3.2.0"
+  spec.required_ruby_version = ">= 3.1.3"
 
   spec.metadata["homepage_uri"] = spec.homepage
 

data/test/instrumentation_test.rb

@@ -67,31 +67,6 @@ class InstrumentationTest < Minitest::Test
     assert_equal OpenTelemetry::Trace::Status::ERROR, span.status.code
   end
 
-  def test_complete_still_works_when_instrumentation_fails
-    stub_request(:post, "https://api.openai.com/v1/chat/completions")
-      .to_return(
-        status: 200,
-        headers: { "Content-Type" => "application/json" },
-        body: {
-          id: "chatcmpl-123",
-          object: "chat.completion",
-          model: "gpt-4o-mini",
-          choices: [{
-            index: 0,
-            message: { role: "assistant", content: "Hello!" },
-            finish_reason: "stop"
-          }],
-          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
-        }.to_json
-      )
-
-    chat = RubyLLM.chat(model: "gpt-4o-mini")
-    chat.define_singleton_method(:tracer) { raise StandardError, "instrumentation bug" }
-
-    response = chat.ask("Hi")
-    assert_equal "Hello!", response.content
-  end
-
   def test_instruments_complete_called_directly
     stub_request(:post, "https://api.openai.com/v1/chat/completions")
       .to_return(
@@ -200,68 +175,6 @@ class InstrumentationTest < Minitest::Test
     assert_equal "function", tool_span.attributes["gen_ai.tool.type"]
   end
 
-  def test_execute_tool_still_works_when_instrumentation_fails
-    calculator = Class.new(RubyLLM::Tool) do
-      def self.name = "calculator"
-      description "Performs math"
-      param :expression, type: "string", desc: "Math expression"
-
-      def execute(expression:)
-        eval(expression).to_s
-      end
-    end
-
-    stub_request(:post, "https://api.openai.com/v1/chat/completions")
-      .to_return(
-        {
-          status: 200,
-          headers: { "Content-Type" => "application/json" },
-          body: {
-            id: "chatcmpl-123",
-            object: "chat.completion",
-            model: "gpt-4o-mini",
-            choices: [{
-              index: 0,
-              message: {
-                role: "assistant",
-                content: nil,
-                tool_calls: [{
-                  id: "call_abc123",
-                  type: "function",
-                  function: { name: "calculator", arguments: '{"expression":"2+2"}' }
-                }]
-              },
-              finish_reason: "tool_calls"
-            }],
-            usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
-          }.to_json
-        },
-        {
-          status: 200,
-          headers: { "Content-Type" => "application/json" },
-          body: {
-            id: "chatcmpl-456",
-            object: "chat.completion",
-            model: "gpt-4o-mini",
-            choices: [{
-              index: 0,
-              message: { role: "assistant", content: "The answer is 4" },
-              finish_reason: "stop"
-            }],
-            usage: { prompt_tokens: 20, completion_tokens: 5, total_tokens: 25 }
-          }.to_json
-        }
-      )
-
-    chat = RubyLLM.chat(model: "gpt-4o-mini")
-    chat.with_tool(calculator)
-
-    chat.define_singleton_method(:tracer) { raise StandardError, "instrumentation bug" }
-
-    response = chat.ask("What is 2+2?")
-    assert_equal "The answer is 4", response.content
-  end
-
   def test_does_not_capture_content_by_default
     stub_request(:post, "https://api.openai.com/v1/chat/completions")
       .to_return(
@@ -332,6 +245,161 @@ class InstrumentationTest < Minitest::Test
     OpenTelemetry::Instrumentation::RubyLLM::Instrumentation.instance.config[:capture_content] = false
   end
 
+  def test_creates_span_for_embedding
+    stub_request(:post, "https://api.openai.com/v1/embeddings")
+      .to_return(
+        status: 200,
+        headers: { "Content-Type" => "application/json" },
+        body: {
+          object: "list",
+          model: "text-embedding-3-small",
+          data: [
+            { object: "embedding", index: 0, embedding: [0.1, 0.2, 0.3] }
+          ],
+          usage: { prompt_tokens: 8, total_tokens: 8 }
+        }.to_json
+      )
+
+    RubyLLM.embed("Hello, world!", model: "text-embedding-3-small")
+
+    spans = EXPORTER.finished_spans
+    assert_equal 1, spans.length
+
+    span = spans.first
+    assert_equal OpenTelemetry::Trace::SpanKind::CLIENT, span.kind
+    assert_equal "embeddings text-embedding-3-small", span.name
+    assert_equal "embeddings", span.attributes["gen_ai.operation.name"]
+    assert_equal "openai", span.attributes["gen_ai.provider.name"]
+    assert_equal "text-embedding-3-small", span.attributes["gen_ai.request.model"]
+    assert_equal "text-embedding-3-small", span.attributes["gen_ai.response.model"]
+    assert_equal 8, span.attributes["gen_ai.usage.input_tokens"]
+    assert_equal 3, span.attributes["gen_ai.embeddings.dimension.count"]
+  end
+
+  def test_records_error_on_embedding_api_failure
+    stub_request(:post, "https://api.openai.com/v1/embeddings")
+      .to_return(status: 500, body: "Internal Server Error")
+
+    assert_raises do
+      RubyLLM.embed("Hello", model: "text-embedding-3-small")
+    end
+
+    spans = EXPORTER.finished_spans
+    span = spans.last
+
+    assert_equal "embeddings text-embedding-3-small", span.name
+    assert span.attributes["error.type"]
+    assert_equal OpenTelemetry::Trace::Status::ERROR, span.status.code
+  end
+
+  def test_with_otel_attributes_sets_span_attributes
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        status: 200,
+        headers: { "Content-Type" => "application/json" },
+        body: {
+          id: "chatcmpl-123",
+          object: "chat.completion",
+          model: "gpt-4o-mini",
+          choices: [{
+            index: 0,
+            message: { role: "assistant", content: "Hello!" },
+            finish_reason: "stop"
+          }],
+          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
+        }.to_json
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.with_otel_attributes(
+      "langfuse.trace.tags" => ["vitamin_d3"],
+      "custom.category" => "supplements"
+    )
+    chat.ask("Hi")
+
+    span = EXPORTER.finished_spans.first
+    assert_equal ["vitamin_d3"], span.attributes["langfuse.trace.tags"]
+    assert_equal "supplements", span.attributes["custom.category"]
+  end
+
+  def test_with_otel_attributes_returns_self_for_chaining
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        status: 200,
+        headers: { "Content-Type" => "application/json" },
+        body: {
+          id: "chatcmpl-123",
+          object: "chat.completion",
+          model: "gpt-4o-mini",
+          choices: [{
+            index: 0,
+            message: { role: "assistant", content: "Hello!" },
+            finish_reason: "stop"
+          }],
+          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
+        }.to_json
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    result = chat.with_otel_attributes("custom.category" => "test")
+
+    assert_same chat, result
+  end
+
+  def test_with_otel_attributes_evaluates_callables
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        status: 200,
+        headers: { "Content-Type" => "application/json" },
+        body: {
+          id: "chatcmpl-123",
+          object: "chat.completion",
+          model: "gpt-4o-mini",
+          choices: [{
+            index: 0,
+            message: { role: "assistant", content: "Hello!" },
+            finish_reason: "stop"
+          }],
+          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
+        }.to_json
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.with_otel_attributes(
+      "custom.last_role" => -> { chat.messages.last&.role.to_s },
+      "custom.static" => "fixed"
+    )
+    chat.ask("Hi")
+
+    span = EXPORTER.finished_spans.first
+    assert_equal "assistant", span.attributes["custom.last_role"]
+    assert_equal "fixed", span.attributes["custom.static"]
+  end
+
+  def test_works_without_otel_attributes
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        status: 200,
+        headers: { "Content-Type" => "application/json" },
+        body: {
+          id: "chatcmpl-123",
+          object: "chat.completion",
+          model: "gpt-4o-mini",
+          choices: [{
+            index: 0,
+            message: { role: "assistant", content: "Hello!" },
+            finish_reason: "stop"
+          }],
+          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
+        }.to_json
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    response = chat.ask("Hi")
+
+    assert_equal "Hello!", response.content
+  end
+
   def test_captures_content_when_enabled_via_env_var
     ENV["OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"] = "true"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: opentelemetry-instrumentation-ruby_llm
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Clarissa Borges
@@ -58,10 +58,12 @@ files:
 - example/trace_demonstration.rb
 - example/trace_demonstration_with_langfuse.rb
 - example/trace_demonstration_with_langfuse_and_tools.rb
+- example/trace_demonstration_with_langfuse_ingredient_search.rb
 - example/trace_demonstration_with_tools.rb
 - lib/opentelemetry-instrumentation-ruby_llm.rb
 - lib/opentelemetry/instrumentation/ruby_llm/instrumentation.rb
 - lib/opentelemetry/instrumentation/ruby_llm/patches/chat.rb
+- lib/opentelemetry/instrumentation/ruby_llm/patches/embedding.rb
 - lib/opentelemetry/instrumentation/ruby_llm/version.rb
 - opentelemetry-instrumentation-ruby_llm.gemspec
 - test/instrumentation_test.rb
@@ -78,7 +80,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.2.0
+      version: 3.1.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="