opentelemetry-instrumentation-ruby_llm 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9e85590961ea3d4ac30828dbc035d692040a2a66758ba6dab9decab2114af3f3
+  data.tar.gz: a8a701fa288ec93109bfb6a4d3fdcc5962c40abc98d9a9269d12a9e430848f78
+SHA512:
+  metadata.gz: 60672653bad15a0e24a97a60ea94433f82df8300370987f656f135a5eb1339b731afbf3644dddbe4a867b3f9970d2b757e06ca7862a4b7d0d36798dff3f9aa58
+  data.tar.gz: d4536fbcba4b6439df8c8a4a30198072eea8cc61265ee1572a5c2b0dbd6f775260381d727d06ebc13604d755dc98f44f24f7ee552d218bb50b2695aec6ad3873

data/.github/workflows/dynamic-readme.yml

@@ -0,0 +1,19 @@
+name: update-templates
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - README.md
+  workflow_dispatch:
+
+jobs:
+  update-templates:
+    permissions:
+      contents: write
+      pull-requests: write
+      pages: write
+    uses: thoughtbot/templates/.github/workflows/dynamic-readme.yaml@main
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}

data/.github/workflows/dynamic-security.yml

@@ -0,0 +1,19 @@
+name: update-security
+
+on:
+  push:
+    paths:
+      - SECURITY.md
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  update-security:
+    permissions:
+      contents: write
+      pull-requests: write
+      pages: write
+    uses: thoughtbot/templates/.github/workflows/dynamic-security.yaml@main
+    secrets:
+      token: ${{ secrets.GITHUB_TOKEN }}

data/.github/workflows/main.yml

@@ -0,0 +1,30 @@
+name: Ruby
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Ruby ${{ matrix.ruby }}
+    strategy:
+      matrix:
+        ruby:
+          - '3.4'
+          - '3.3'
+          - '3.2'
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - name: Run tests
+        run: bundle exec rake

data/.gitignore

@@ -0,0 +1 @@
+*.gem

data/CODEOWNERS

@@ -0,0 +1,15 @@
+# Lines starting with '#' are comments.
+# Each line is a file pattern followed by one or more owners.
+
+# More details are here: https://help.github.com/articles/about-codeowners/
+
+# The '*' pattern is global owners.
+
+# Order is important. The last matching pattern has the most precedence.
+# The folders are ordered as follows:
+
+# In each subsection folders are ordered first by depth, then alphabetically.
+# This should make it easy to add new rules without breaking existing ones.
+
+# Global rule:
+* @clarissalimab

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,6 @@
+# Code of conduct
+
+By participating in this project, you agree to abide by the
+[thoughtbot code of conduct][1].
+
+[1]: https://thoughtbot.com/open-source-code-of-conduct

data/Gemfile

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec
+
+gem "ruby_llm"
+gem "opentelemetry-sdk"
+gem "opentelemetry-exporter-otlp"
+
+group :test do
+  gem "minitest"
+  gem "webmock"
+  gem "rake"
+end

data/Gemfile.lock

@@ -0,0 +1,105 @@
+PATH
+  remote: .
+  specs:
+    opentelemetry-instrumentation-ruby_llm (0.1.0)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.23)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.8)
+      public_suffix (>= 2.0.2, < 8.0)
+    base64 (0.3.0)
+    bigdecimal (4.0.1)
+    crack (1.0.1)
+      bigdecimal
+      rexml
+    event_stream_parser (1.0.0)
+    faraday (2.14.0)
+      faraday-net_http (>= 2.0, < 3.5)
+      json
+      logger
+    faraday-multipart (1.2.0)
+      multipart-post (~> 2.0)
+    faraday-net_http (3.4.2)
+      net-http (~> 0.5)
+    faraday-retry (2.4.0)
+      faraday (~> 2.0)
+    google-protobuf (4.33.4)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.33.4-arm64-darwin)
+      bigdecimal
+      rake (>= 13)
+    googleapis-common-protos-types (1.22.0)
+      google-protobuf (~> 4.26)
+    hashdiff (1.2.1)
+    json (2.18.0)
+    logger (1.7.0)
+    marcel (1.1.0)
+    minitest (6.0.1)
+      prism (~> 1.5)
+    multipart-post (2.4.1)
+    net-http (0.9.1)
+      uri (>= 0.11.1)
+    opentelemetry-api (1.7.0)
+    opentelemetry-common (0.23.0)
+      opentelemetry-api (~> 1.0)
+    opentelemetry-exporter-otlp (0.31.1)
+      google-protobuf (>= 3.18)
+      googleapis-common-protos-types (~> 1.3)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-sdk (~> 1.10)
+      opentelemetry-semantic_conventions
+    opentelemetry-instrumentation-base (0.25.0)
+      opentelemetry-api (~> 1.7)
+      opentelemetry-common (~> 0.21)
+      opentelemetry-registry (~> 0.1)
+    opentelemetry-registry (0.4.0)
+      opentelemetry-api (~> 1.1)
+    opentelemetry-sdk (1.10.0)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-registry (~> 0.2)
+      opentelemetry-semantic_conventions
+    opentelemetry-semantic_conventions (1.36.0)
+      opentelemetry-api (~> 1.0)
+    prism (1.9.0)
+    public_suffix (7.0.2)
+    rake (13.3.1)
+    rexml (3.4.4)
+    ruby_llm (1.11.0)
+      base64
+      event_stream_parser (~> 1)
+      faraday (>= 1.10.0)
+      faraday-multipart (>= 1)
+      faraday-net_http (>= 1)
+      faraday-retry (>= 1)
+      marcel (~> 1.0)
+      ruby_llm-schema (~> 0.2.1)
+      zeitwerk (~> 2)
+    ruby_llm-schema (0.2.5)
+    uri (1.1.1)
+    webmock (3.26.1)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+    zeitwerk (2.7.4)
+
+PLATFORMS
+  arm64-darwin-24
+  ruby
+
+DEPENDENCIES
+  minitest
+  opentelemetry-exporter-otlp
+  opentelemetry-instrumentation-ruby_llm!
+  opentelemetry-sdk
+  rake
+  ruby_llm
+  webmock
+
+BUNDLED WITH
+   2.7.1

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) Clarissa Borges and thoughtbot, inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,69 @@
+# OpenTelemetry RubyLLM Instrumentation
+
+OpenTelemetry instrumentation for [RubyLLM](https://rubyllm.com).
+
+## How do I get started?
+
+Install the gem using:
+
+```sh
+gem opentelemetry-instrumentation-ruby_llm
+```
+
+Or, if you use [bundler](https://bundler.io/), include `opentelemetry-instrumentation-ruby_llm` in your `Gemfile`.
+
+## Usage
+
+To use the instrumentation, call `use` with the name of the instrumentation:
+
+```ruby
+OpenTelemetry::SDK.configure do |c|
+  c.use 'OpenTelemetry::Instrumentation::RubyLLM'
+end
+```
+
+Alternatively, you can also call `use_all` to install all the available instrumentation.
+
+```ruby
+OpenTelemetry::SDK.configure do |c|
+  c.use_all
+end
+```
+
+## What's traced?
+
+| Feature | Status |
+|---------|--------|
+| Chat completions | Supported |
+| Tool calls | Supported |
+| Error handling | Supported |
+| Conversation tracking (`gen_ai.conversation.id`) | Planned |
+| Opt-in input/output content capture | Planned |
+| System instructions capture | Planned |
+| Embeddings | Planned |
+| Streaming | Planned |
+
+This gem follows the [OpenTelemetry GenAI Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-spans/).
+
+## License
+
+Copyright (c) Clarissa Borges and thoughtbot, inc.
+
+This gem is free software and may be redistributed under the terms specified in the [LICENSE](LICENSE) file.
+
+<!-- START /templates/footer.md -->
+## About thoughtbot
+
+![thoughtbot](https://thoughtbot.com/thoughtbot-logo-for-readmes.svg)
+
+This repo is maintained and funded by thoughtbot, inc.
+The names and logos for thoughtbot are trademarks of thoughtbot, inc.
+
+We love open source software!
+See [our other projects][community].
+We are [available for hire][hire].
+
+[community]: https://thoughtbot.com/community?utm_source=github
+[hire]: https://thoughtbot.com/hire-us?utm_source=github
+
+<!-- END /templates/footer.md -->

data/Rakefile

@@ -0,0 +1,8 @@
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.pattern = "test/**/*_test.rb"
+end
+
+task default: :test

data/SECURITY.md

@@ -0,0 +1,20 @@
+<!-- START /templates/security.md -->
+# Security Policy
+
+## Supported Versions
+
+Only the the latest version of this project is supported at a given time. If
+you find a security issue with an older version, please try updating to the
+latest version first.
+
+If for some reason you can't update to the latest version, please let us know
+your reasons so that we can have a better understanding of your situation.
+
+## Reporting a Vulnerability
+
+For security inquiries or vulnerability reports, visit
+<https://thoughtbot.com/security>.
+
+If you have any suggestions to improve this policy, visit <https://thoughtbot.com/security>.
+
+<!-- END /templates/security.md -->

data/example/trace_demonstration.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "bundler/inline"
+
+gemfile(true) do
+  source "https://rubygems.org"
+  gem "ruby_llm"
+  gem "opentelemetry-api"
+  gem "opentelemetry-sdk"
+  gem "opentelemetry-instrumentation-ruby_llm", path: "../"
+end
+
+ENV["OTEL_TRACES_EXPORTER"] ||= "console"
+
+OpenTelemetry::SDK.configure do |c|
+  c.use "OpenTelemetry::Instrumentation::RubyLLM"
+end
+
+RubyLLM.configure do |c|
+  c.openai_api_key = ENV["OPENAI_API_KEY"]
+  c.default_model = "gpt-5-nano"
+end
+
+chat = RubyLLM.chat
+response = chat.ask("What is the meaning of life?")
+puts "\nResponse: #{response.content}"
+
+# This line is only necessary in short-lived scripts. In a long-running application, spans will be flushed automatically.
+OpenTelemetry.tracer_provider.force_flush

data/example/trace_demonstration_with_langfuse.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "bundler/inline"
+
+gemfile(true) do
+  source "https://rubygems.org"
+  gem "ruby_llm"
+  gem "opentelemetry-api"
+  gem "opentelemetry-sdk"
+  gem "opentelemetry-exporter-otlp"
+  gem "opentelemetry-instrumentation-ruby_llm", path: "../"
+  gem "base64"
+end
+
+require "base64"
+
+credentials = Base64.strict_encode64("#{ENV['LANGFUSE_PUBLIC_KEY']}:#{ENV['LANGFUSE_SECRET_KEY']}")
+
+OpenTelemetry::SDK.configure do |c|
+  c.service_name = "ruby_llm-demo"
+  c.add_span_processor(
+    OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
+      OpenTelemetry::Exporter::OTLP::Exporter.new(
+        endpoint: "https://us.cloud.langfuse.com/api/public/otel/v1/traces",
+        headers: { "Authorization" => "Basic #{credentials}" }
+      )
+    )
+  )
+  c.use "OpenTelemetry::Instrumentation::RubyLLM"
+end
+
+RubyLLM.configure do |c|
+  c.openai_api_key = ENV["OPENAI_API_KEY"]
+  c.default_model = "gpt-5-nano"
+end
+
+chat = RubyLLM.chat
+response = chat.ask("What is the meaning of life?")
+puts "\nResponse: #{response.content}"
+
+# This line is only necessary in short-lived scripts. In a long-running application, spans will be flushed automatically.
+OpenTelemetry.tracer_provider.force_flush

data/example/trace_demonstration_with_langfuse_and_tools.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "bundler/inline"
+
+gemfile(true) do
+  source "https://rubygems.org"
+  gem "ruby_llm"
+  gem "opentelemetry-api"
+  gem "opentelemetry-sdk"
+  gem "opentelemetry-exporter-otlp"
+  gem "opentelemetry-instrumentation-ruby_llm", path: "../"
+  gem "base64"
+end
+
+require "base64"
+
+credentials = Base64.strict_encode64("#{ENV['LANGFUSE_PUBLIC_KEY']}:#{ENV['LANGFUSE_SECRET_KEY']}")
+
+OpenTelemetry::SDK.configure do |c|
+  c.service_name = "ruby_llm-demo"
+  c.add_span_processor(
+    OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
+      OpenTelemetry::Exporter::OTLP::Exporter.new(
+        endpoint: "https://us.cloud.langfuse.com/api/public/otel/v1/traces",
+        headers: { "Authorization" => "Basic #{credentials}" }
+      )
+    )
+  )
+  c.use "OpenTelemetry::Instrumentation::RubyLLM"
+end
+
+RubyLLM.configure do |c|
+  c.openai_api_key = ENV["OPENAI_API_KEY"]
+  c.default_model = "gpt-5-nano"
+end
+
+class Calculator < RubyLLM::Tool
+  description "Performs basic math calculations"
+  param :expression, type: "string", desc: "Math expression to evaluate"
+
+  def execute(expression:)
+    eval(expression).to_s
+  end
+end
+
+chat = RubyLLM.chat
+chat.with_tool(Calculator)
+response = chat.ask("Use the calculator tool to compute 123 * 456")
+puts "\nResponse: #{response.content}"
+
+# This line is only necessary in short-lived scripts. In a long-running application, spans will be flushed automatically.
+OpenTelemetry.tracer_provider.force_flush

data/example/trace_demonstration_with_tools.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "bundler/inline"
+
+gemfile(true) do
+  source "https://rubygems.org"
+  gem "ruby_llm"
+  gem "opentelemetry-api"
+  gem "opentelemetry-sdk"
+  gem "opentelemetry-instrumentation-ruby_llm", path: "../"
+end
+
+ENV["OTEL_TRACES_EXPORTER"] ||= "console"
+
+OpenTelemetry::SDK.configure do |c|
+  c.use "OpenTelemetry::Instrumentation::RubyLLM"
+end
+
+RubyLLM.configure do |c|
+  c.openai_api_key = ENV["OPENAI_API_KEY"]
+  c.default_model = "gpt-5-nano"
+end
+
+class Calculator < RubyLLM::Tool
+  description "Performs basic math calculations"
+  param :expression, type: "string", desc: "Math expression to evaluate"
+
+  def execute(expression:)
+    eval(expression).to_s
+  end
+end
+
+chat = RubyLLM.chat
+chat.with_tool(Calculator)
+response = chat.ask("What is 123 * 456?")
+puts "\nResponse: #{response.content}"
+
+# This line is only necessary in short-lived scripts. In a long-running application, spans will be flushed automatically.
+OpenTelemetry.tracer_provider.force_flush

data/lib/opentelemetry/instrumentation/ruby_llm/instrumentation.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module OpenTelemetry
+  module Instrumentation
+    module RubyLLM
+      class Instrumentation < OpenTelemetry::Instrumentation::Base
+        instrumentation_name "OpenTelemetry::Instrumentation::RubyLLM"
+        instrumentation_version VERSION
+
+        present do
+          defined?(::RubyLLM)
+        end
+
+        install do |_config|
+          require_relative "patches/chat"
+          ::RubyLLM::Chat.prepend(Patches::Chat)
+        end
+      end
+    end
+  end
+end

data/lib/opentelemetry/instrumentation/ruby_llm/patches/chat.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module OpenTelemetry
+  module Instrumentation
+    module RubyLLM
+      module Patches
+        module Chat
+          def ask(message, &block)
+            provider = @model&.provider || "unknown"
+            model_id = @model&.id || "unknown"
+
+            attributes = {
+              "gen_ai.operation.name" => "chat",
+              "gen_ai.provider.name" => provider,
+              "gen_ai.request.model" => model_id,
+            }
+
+            tracer.in_span("chat #{model_id}", attributes: attributes, kind: OpenTelemetry::Trace::SpanKind::CLIENT) do |span|
+              begin
+                result = super
+
+                if @messages.last
+                  response = @messages.last
+                  span.set_attribute("gen_ai.response.model", response.model_id) if response.model_id
+                  span.set_attribute("gen_ai.usage.input_tokens", response.input_tokens) if response.input_tokens
+                  span.set_attribute("gen_ai.usage.output_tokens", response.output_tokens) if response.output_tokens
+                  span.set_attribute("gen_ai.request.temperature", @temperature) if @temperature
+                end
+
+                result
+              rescue => e
+                span.record_exception(e)
+                span.status = OpenTelemetry::Trace::Status.error(e.message)
+                span.set_attribute("error.type", e.class.name)
+                raise
+              end
+            end
+          rescue StandardError => e
+            OpenTelemetry.handle_error(exception: e)
+            super
+          end
+
+          def execute_tool(tool_call)
+            attributes = {
+              "gen_ai.tool.name" => tool_call.name,
+              "gen_ai.tool.call.id" => tool_call.id,
+              "gen_ai.tool.call.arguments" => tool_call.arguments.to_json,
+              "gen_ai.tool.type" => "function"
+            }
+
+            tracer.in_span("execute_tool #{tool_call.name}", attributes: attributes, kind: OpenTelemetry::Trace::SpanKind::INTERNAL) do |span|
+              result = super
+              result_str = result.is_a?(::RubyLLM::Tool::Halt) ? result.content.to_s : result.to_s
+              span.set_attribute("gen_ai.tool.call.result", result_str[0..500])
+              result
+            end
+          rescue StandardError => e
+            OpenTelemetry.handle_error(exception: e)
+            super
+          end
+
+          private
+
+          def tracer
+            RubyLLM::Instrumentation.instance.tracer
+          end
+        end
+      end
+    end
+  end
+end

data/lib/opentelemetry/instrumentation/ruby_llm/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module OpenTelemetry
+  module Instrumentation
+    module RubyLLM
+      VERSION = "0.1.0"
+    end
+  end
+end

data/lib/opentelemetry-instrumentation-ruby_llm.rb

@@ -0,0 +1,5 @@
+require "opentelemetry-api"
+require "opentelemetry-instrumentation-base"
+
+require_relative "opentelemetry/instrumentation/ruby_llm/version"
+require_relative "opentelemetry/instrumentation/ruby_llm/instrumentation"

data/opentelemetry-instrumentation-ruby_llm.gemspec

@@ -0,0 +1,23 @@
+require_relative "lib/opentelemetry/instrumentation/ruby_llm/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "opentelemetry-instrumentation-ruby_llm"
+  spec.version       = OpenTelemetry::Instrumentation::RubyLLM::VERSION
+  spec.authors       = ["Clarissa Borges"]
+  spec.email         = ["cborges@thoughtbot.com"]
+  spec.license       = "MIT"
+
+  spec.summary     = "OpenTelemetry instrumentation for RubyLLM"
+  spec.description = "Adds OpenTelemetry tracing to RubyLLM chat operations"
+  spec.homepage    = "https://github.com/thoughtbot/opentelemetry-instrumentation-ruby_llm"
+
+  spec.required_ruby_version = ">= 3.2.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+
+  spec.files = `git ls-files`.split("\n")
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "opentelemetry-api", "~> 1.0"
+  spec.add_dependency "opentelemetry-instrumentation-base", "~> 0.23"
+end

data/test/instrumentation_test.rb

@@ -0,0 +1,231 @@
+require "test_helper"
+
+class InstrumentationTest < Minitest::Test
+  def setup
+    EXPORTER.reset
+
+    RubyLLM.configure do |c|
+      c.openai_api_key = "fake-key-for-testing"
+    end
+  end
+
+  def test_creates_span_with_attributes
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        status: 200,
+        headers: { "Content-Type" => "application/json" },
+        body: {
+          id: "chatcmpl-123",
+          object: "chat.completion",
+          model: "gpt-4o-mini",
+          choices: [
+            {
+              index: 0,
+              message: { role: "assistant", content: "Hello, world!" },
+              finish_reason: "stop"
+            }
+          ],
+          usage: {
+            prompt_tokens: 10,
+            completion_tokens: 5,
+            total_tokens: 15
+          }
+        }.to_json
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.ask("Hi")
+
+    spans = EXPORTER.finished_spans
+    assert_equal 1, spans.length
+
+    span = spans.first
+    assert_equal OpenTelemetry::Trace::SpanKind::CLIENT, span.kind
+    assert_equal "chat gpt-4o-mini", span.name
+    assert_equal "openai", span.attributes["gen_ai.provider.name"]
+    assert_equal "gpt-4o-mini", span.attributes["gen_ai.request.model"]
+    assert_equal "chat", span.attributes["gen_ai.operation.name"]
+    assert_equal 10, span.attributes["gen_ai.usage.input_tokens"]
+    assert_equal 5, span.attributes["gen_ai.usage.output_tokens"]
+  end
+
+  def test_records_error_on_api_failure
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(status: 500, body: "Internal Server Error")
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+
+    assert_raises do
+      chat.ask("Hi")
+    end
+
+    spans = EXPORTER.finished_spans
+    span = spans.last
+
+    assert_equal "chat gpt-4o-mini", span.name
+    assert span.attributes["error.type"]
+    assert_equal OpenTelemetry::Trace::Status::ERROR, span.status.code
+  end
+
+  def test_ask_still_works_when_instrumentation_fails
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        status: 200,
+        headers: { "Content-Type" => "application/json" },
+        body: {
+          id: "chatcmpl-123",
+          object: "chat.completion",
+          model: "gpt-4o-mini",
+          choices: [{
+            index: 0,
+            message: { role: "assistant", content: "Hello!" },
+            finish_reason: "stop"
+          }],
+          usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
+        }.to_json
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.define_singleton_method(:tracer) { raise StandardError, "instrumentation bug" }
+
+    response = chat.ask("Hi")
+    assert_equal "Hello!", response.content
+  end
+
+  def test_creates_span_for_tool_call
+    calculator = Class.new(RubyLLM::Tool) do
+      def self.name = "calculator"
+      description "Performs math"
+      param :expression, type: "string", desc: "Math expression"
+
+      def execute(expression:)
+        eval(expression).to_s
+      end
+    end
+
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        {
+          status: 200,
+          headers: { "Content-Type" => "application/json" },
+          body: {
+            id: "chatcmpl-123",
+            object: "chat.completion",
+            model: "gpt-4o-mini",
+            choices: [{
+              index: 0,
+              message: {
+                role: "assistant",
+                content: nil,
+                tool_calls: [{
+                  id: "call_abc123",
+                  type: "function",
+                  function: { name: "calculator", arguments: '{"expression":"2+2"}' }
+                }]
+              },
+              finish_reason: "tool_calls"
+            }],
+            usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
+          }.to_json
+        },
+        {
+          status: 200,
+          headers: { "Content-Type" => "application/json" },
+          body: {
+            id: "chatcmpl-456",
+            object: "chat.completion",
+            model: "gpt-4o-mini",
+            choices: [{
+              index: 0,
+              message: { role: "assistant", content: "The answer is 4" },
+              finish_reason: "stop"
+            }],
+            usage: { prompt_tokens: 20, completion_tokens: 5, total_tokens: 25 }
+          }.to_json
+        }
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.with_tool(calculator)
+    chat.ask("What is 2+2?")
+
+    spans = EXPORTER.finished_spans
+
+    tool_spans = spans.select { |s| s.name.start_with?("execute_tool ") }
+    chat_spans = spans.select { |s| s.name.include?("chat ") }
+
+    assert_equal 1, tool_spans.length
+    assert_equal 1, chat_spans.length
+
+    tool_span = tool_spans.first
+    assert_equal OpenTelemetry::Trace::SpanKind::INTERNAL, tool_span.kind
+    assert_equal "execute_tool calculator", tool_span.name
+    assert_equal "calculator", tool_span.attributes["gen_ai.tool.name"]
+    assert_equal '{"expression":"2+2"}', tool_span.attributes["gen_ai.tool.call.arguments"]
+    assert_equal "4", tool_span.attributes["gen_ai.tool.call.result"]
+    assert_equal "call_abc123", tool_span.attributes["gen_ai.tool.call.id"]
+    assert_equal "function", tool_span.attributes["gen_ai.tool.type"]
+  end
+
+  def test_execute_tool_still_works_when_instrumentation_fails
+    calculator = Class.new(RubyLLM::Tool) do
+      def self.name = "calculator"
+      description "Performs math"
+      param :expression, type: "string", desc: "Math expression"
+
+      def execute(expression:)
+        eval(expression).to_s
+      end
+    end
+
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        {
+          status: 200,
+          headers: { "Content-Type" => "application/json" },
+          body: {
+            id: "chatcmpl-123",
+            object: "chat.completion",
+            model: "gpt-4o-mini",
+            choices: [{
+              index: 0,
+              message: {
+                role: "assistant",
+                content: nil,
+                tool_calls: [{
+                  id: "call_abc123",
+                  type: "function",
+                  function: { name: "calculator", arguments: '{"expression":"2+2"}' }
+                }]
+              },
+              finish_reason: "tool_calls"
+            }],
+            usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
+          }.to_json
+        },
+        {
+          status: 200,
+          headers: { "Content-Type" => "application/json" },
+          body: {
+            id: "chatcmpl-456",
+            object: "chat.completion",
+            model: "gpt-4o-mini",
+            choices: [{
+              index: 0,
+              message: { role: "assistant", content: "The answer is 4" },
+              finish_reason: "stop"
+            }],
+            usage: { prompt_tokens: 20, completion_tokens: 5, total_tokens: 25 }
+          }.to_json
+        }
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.with_tool(calculator)
+
+    chat.define_singleton_method(:tracer) { raise StandardError, "instrumentation bug" }
+
+    response = chat.ask("What is 2+2?")
+    assert_equal "The answer is 4", response.content
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,15 @@
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+
+require "minitest/autorun"
+require "webmock/minitest"
+require "ruby_llm"
+require "opentelemetry/sdk"
+require "opentelemetry-instrumentation-ruby_llm"
+
+EXPORTER = OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter.new
+span_processor = OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(EXPORTER)
+
+OpenTelemetry::SDK.configure do |c|
+  c.add_span_processor(span_processor)
+  c.use "OpenTelemetry::Instrumentation::RubyLLM"
+end

metadata

@@ -0,0 +1,92 @@
+--- !ruby/object:Gem::Specification
+name: opentelemetry-instrumentation-ruby_llm
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Clarissa Borges
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: opentelemetry-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: opentelemetry-instrumentation-base
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.23'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.23'
+description: Adds OpenTelemetry tracing to RubyLLM chat operations
+email:
+- cborges@thoughtbot.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/dynamic-readme.yml"
+- ".github/workflows/dynamic-security.yml"
+- ".github/workflows/main.yml"
+- ".gitignore"
+- CODEOWNERS
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.md
+- Rakefile
+- SECURITY.md
+- example/trace_demonstration.rb
+- example/trace_demonstration_with_langfuse.rb
+- example/trace_demonstration_with_langfuse_and_tools.rb
+- example/trace_demonstration_with_tools.rb
+- lib/opentelemetry-instrumentation-ruby_llm.rb
+- lib/opentelemetry/instrumentation/ruby_llm/instrumentation.rb
+- lib/opentelemetry/instrumentation/ruby_llm/patches/chat.rb
+- lib/opentelemetry/instrumentation/ruby_llm/version.rb
+- opentelemetry-instrumentation-ruby_llm.gemspec
+- test/instrumentation_test.rb
+- test/test_helper.rb
+homepage: https://github.com/thoughtbot/opentelemetry-instrumentation-ruby_llm
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/thoughtbot/opentelemetry-instrumentation-ruby_llm
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.7.1
+specification_version: 4
+summary: OpenTelemetry instrumentation for RubyLLM
+test_files: []