ruby_llm-instrumentation 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 67942ad10a363e6e530277085f6750ea53eb547e88df806f19ddbbebef286adc
+  data.tar.gz: 4a2208687976b6744d1cbe6e19aa03d0a5bf9d080f4597ad1341e8d9cf590a4b
+SHA512:
+  metadata.gz: f9190dea5c2c5d4c8824f4e93c278a527c770204370b93a0e69d622cd09369597674ceb1dc6fa72e499b5e6858afb95a83d1582d8ed67fa9cac44b53fd94e95f
+  data.tar.gz: 254a5922c0a6fa80c8d907de2575e8d43cff803f0d4c9c1892d3285ef16159bf53abb61e73a49552345bdd83566b9837c5509327cad344cce9f99e41bfce2ddf

data/README.md

@@ -0,0 +1,133 @@
+# RubyLLM::Instrumentation
+
+Rails instrumentation for RubyLLM.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem "ruby_llm-instrumentation"
+```
+
+And then execute:
+
+```bash
+$ bundle
+```
+
+Now, RubyLLM will instrument all calls to your configured LLM.
+
+## Usage
+
+RubyLLM::Instrumentation uses ActiveSupport::Notifications to publish events. You can subscribe to these events to build custom monitoring, logging, or analytics:
+
+```ruby
+# Subscribe to all LLM events
+ActiveSupport::Notifications.subscribe(/ruby_llm/) do |name, start, finish, id, payload|
+  duration = finish - start
+
+  Rails.logger.info "LLM Call: #{payload[:provider]}/#{payload[:model]}"
+  Rails.logger.info "Duration: #{duration}ms"
+  Rails.logger.info "Input tokens: #{payload[:input_tokens]}"
+  Rails.logger.info "Output tokens: #{payload[:output_tokens]}"
+end
+```
+
+## Available Events
+
+### RubyLLM::Chat
+
+#### complete_chat.ruby_llm
+
+Triggered when `#ask` is called.
+
+| Key                   | Value                                   |
+| --------------------- | --------------------------------------- |
+| provider              | Provider slug                           |
+| model                 | Model ID                                |
+| streaming             | Whether streaming was used              |
+| chat                  | The chat, a RubyLLM::Chat object        |
+| response              | The response, a RubyLLM::Message object |
+| input_tokens          | Input tokens consumed                   |
+| output_tokens         | Output tokens consumed                  |
+| cached_tokens         | Cache reads tokens (if supported)       |
+| cache_creation_tokens | Cache write tokens (if supported)       |
+
+#### execute_tool.ruby_llm
+
+Triggered when `#execute_tool` is called.
+
+| Key       | Value                                               |
+| --------- | --------------------------------------------------- |
+| provider  | Provider slug                                       |
+| model     | Model ID                                            |
+| tool_call | The tool call, a RubyLLM::ToolCall object           |
+| tool_name | The tool name                                       |
+| arguments | The arguments                                       |
+| chat      | The chat, a RubyLLM::Chat instance                  |
+| halted    | Indicates if the tool stopped the conversation loop |
+
+### RubyLLM::Embedding
+
+#### embed_text.ruby_llm
+
+Triggered when `.embed` is called.
+
+| Key          | Value                                                          |
+| ------------ | -------------------------------------------------------------- |
+| provider     | Provider slug                                                  |
+| embedding    | The embedding, a RubyLLM::Embedding object                     |
+| model        | Model ID                                                       |
+| dimensions   | Number of embedding dimensions (or array of sizes if multiple) |
+| input_tokens | Input tokens consumed                                          |
+| vector_count | Number of vectors generated                                    |
+
+### RubyLLM::Image
+
+#### paint_image.ruby_llm
+
+Triggered when `.paint` is called.
+
+| Key      | Value                                        |
+| -------- | -------------------------------------------- |
+| provider | Provider slug                                |
+| size     | Image dimensions                             |
+| image    | The inage generated, a RubyLLM::Image object |
+| model    | Model ID                                     |
+
+### RubyLLM::Moderation
+
+#### moderate_text.ruby_llm
+
+Triggered when `.moderate` is called.
+
+| Key        | Value                                        |
+| ---------- | -------------------------------------------- |
+| provider   | Provider slug                                |
+| moderation | The moderation, a RubyLLM::Moderation object |
+| model      | Model ID                                     |
+| flagged    | Whether the text was flagged                 |
+
+### RubyLLM::Transcription
+
+#### transcribe_audio.ruby_llm
+
+Triggered when `.transcribe` is called.
+
+| Key           | Value                                              |
+| ------------- | -------------------------------------------------- |
+| provider      | Provider slug                                      |
+| transcription | The transcription, a RubyLLM::Transcription object |
+| model         | Model ID                                           |
+| input_tokens  | Input tokens consumed                              |
+| output_tokens | Output tokens consumed                             |
+| duration      | Audio duration in seconds (if available)           |
+
+## Contributing
+
+You can open an issue or a PR in GitHub.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/setup"
+
+require "bundler/gem_tasks"

data/lib/ruby_llm/instrumentation/chat.rb

@@ -0,0 +1,49 @@
+module RubyLLM
+  module Instrumentation
+    module Chat
+      extend ActiveSupport::Concern
+
+      included do
+        alias_method :original_complete, :complete
+        def complete(&)
+          raw_payload = {
+            provider: @provider.slug,
+            model: @model.id,
+            streaming: block_given?
+          }
+
+          ActiveSupport::Notifications.instrument("complete_chat.ruby_llm", raw_payload) do |payload|
+            original_complete(&).tap do |response|
+              payload[:response] = response
+              %i[input_tokens output_tokens cached_tokens cache_creation_tokens].each do |field|
+                value = response.public_send(field)
+                payload[field] = value unless value.nil?
+              end
+            end
+          ensure
+            payload[:chat] = self
+          end
+        end
+
+        alias_method :original_execute_tool, :execute_tool
+        def execute_tool(tool_call)
+          raw_payload = {
+            provider: @provider.slug,
+            model: @model.id,
+            tool_call: tool_call,
+            tool_name: tool_call.name,
+            arguments: tool_call.arguments
+          }
+
+          ActiveSupport::Notifications.instrument("execute_tool.ruby_llm", raw_payload) do |payload|
+            original_execute_tool(tool_call).tap do |response|
+              payload[:halted] = response.is_a?(Tool::Halt)
+            end
+          ensure
+            payload[:chat] = self
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/instrumentation/embedding.rb

@@ -0,0 +1,30 @@
+module RubyLLM
+  module Instrumentation
+    module Embedding
+      extend ActiveSupport::Concern
+
+      included do
+        class << self
+          alias_method :original_embed, :embed
+          def embed(text, model: nil, provider: nil, assume_model_exists: false, context: nil, dimensions: nil)
+            raw_payload = {
+              provider:
+            }
+
+            ActiveSupport::Notifications.instrument("embed_text.ruby_llm", raw_payload) do |payload|
+              original_embed(text, model:, provider:, assume_model_exists:, context:, dimensions:).tap do |response|
+                payload[:embedding] = response
+                payload[:model] = response.model
+                payload[:input_tokens] = response.input_tokens unless response.input_tokens.nil?
+
+                # response.vectors is an array of floats, or an array of an array of floats
+                payload[:vector_count] = response.vectors.first.is_a?(Array) ? response.vectors.size : 1
+                payload[:dimensions] = response.vectors.first.is_a?(Array) ? response.vectors.map(&:size) : response.vectors.size
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/instrumentation/image.rb

@@ -0,0 +1,26 @@
+module RubyLLM
+  module Instrumentation
+    module Image
+      extend ActiveSupport::Concern
+
+      included do
+        class << self
+          alias_method :original_paint, :paint
+          def paint(prompt, model: nil, provider: nil, assume_model_exists: false, size: "1024x1024", context: nil)
+            raw_payload = {
+              provider:,
+              size:
+            }
+
+            ActiveSupport::Notifications.instrument("paint_image.ruby_llm", raw_payload) do |payload|
+              original_paint(prompt, model:, provider:, assume_model_exists:, size:, context:).tap do |response|
+                payload[:image] = response
+                payload[:model] = response.model
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/instrumentation/moderation.rb

@@ -0,0 +1,26 @@
+module RubyLLM
+  module Instrumentation
+    module Moderation
+      extend ActiveSupport::Concern
+
+      included do
+        class << self
+          alias_method :original_moderate, :moderate
+          def moderate(input, model: nil, provider: nil, assume_model_exists: false, context: nil)
+            raw_payload = {
+              provider:
+            }
+
+            ActiveSupport::Notifications.instrument("moderate_text.ruby_llm", raw_payload) do |payload|
+              original_moderate(input, model:, provider:, assume_model_exists:, context:).tap do |response|
+                payload[:moderation] = response
+                payload[:model] = response.model
+                payload[:flagged] = response.flagged?
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/instrumentation/railtie.rb

@@ -0,0 +1,23 @@
+module RubyLLM
+  module Instrumentation
+    class Railtie < ::Rails::Railtie
+      INFLECTION_OVERRIDES = { "ruby_llm" => "RubyLLM" }.freeze
+
+      initializer "ruby_llm_instrumentation.inflector", after: "ruby_llm.inflections", before: :set_autoload_paths do
+        ActiveSupport::Inflector.inflections(:en) do |inflections|
+          # The RubyLLM gem registers "RubyLLM" as an acronym in its railtie,
+          # which breaks underscore conversion (RubyLLM.underscore => "rubyllm").
+          # We need to remove it and use "LLM" as an acronym instead for proper conversion:
+          # * "ruby_llm".camelize => "RubyLLM" (not "RubyLlm")
+          # * "RubyLLM".underscore => "ruby_llm" (not "rubyllm")
+          inflections.acronyms.delete("rubyllm")
+          inflections.acronym("LLM")
+        end
+
+        Rails.autoloaders.each do |loader|
+          loader.inflector.inflect(INFLECTION_OVERRIDES)
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/instrumentation/transcription.rb

@@ -0,0 +1,29 @@
+module RubyLLM
+  module Instrumentation
+    module Transcription
+      extend ActiveSupport::Concern
+
+      included do
+        class << self
+          alias_method :original_transcribe, :transcribe
+          def transcribe(audio_file, **kwargs)
+            raw_payload = {
+              provider: kwargs[:provider]
+            }
+
+            ActiveSupport::Notifications.instrument("transcribe_audio.ruby_llm", raw_payload) do |payload|
+              original_transcribe(audio_file, **kwargs).tap do |response|
+                payload[:transcription] = response
+                payload[:model] = response.model
+                %i[input_tokens output_tokens duration].each do |field|
+                  value = response.public_send(field)
+                  payload[field] = value unless value.nil?
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ruby_llm/instrumentation/version.rb

@@ -0,0 +1,5 @@
+module RubyLLM
+  module Instrumentation
+    VERSION = "0.1.1"
+  end
+end

data/lib/ruby_llm/instrumentation.rb

@@ -0,0 +1,19 @@
+require "ruby_llm"
+require "ruby_llm/instrumentation/version"
+require "ruby_llm/instrumentation/railtie"
+
+module RubyLLM
+  module Instrumentation
+    autoload :Chat, "ruby_llm/instrumentation/chat"
+    autoload :Embedding, "ruby_llm/instrumentation/embedding"
+    autoload :Image, "ruby_llm/instrumentation/image"
+    autoload :Transcription, "ruby_llm/instrumentation/transcription"
+    autoload :Moderation, "ruby_llm/instrumentation/moderation"
+  end
+end
+
+RubyLLM::Chat.include RubyLLM::Instrumentation::Chat
+RubyLLM::Embedding.include RubyLLM::Instrumentation::Embedding
+RubyLLM::Image.include RubyLLM::Instrumentation::Image
+RubyLLM::Transcription.include RubyLLM::Instrumentation::Transcription
+RubyLLM::Moderation.include RubyLLM::Instrumentation::Moderation

data/lib/tasks/ruby_llm/instrumentation_tasks.rake

@@ -0,0 +1,4 @@
+# desc "Explaining what the task does"
+# task :ruby_llm_instrumentation do
+#   # Task goes here
+# end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: ruby_llm-instrumentation
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Patricio Mac Adden
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 7.0.0
+- !ruby/object:Gem::Dependency
+  name: ruby_llm
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Rails instrumentation for RubyLLM
+email:
+- patricio.macadden@sinaptia.dev
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- Rakefile
+- lib/ruby_llm/instrumentation.rb
+- lib/ruby_llm/instrumentation/chat.rb
+- lib/ruby_llm/instrumentation/embedding.rb
+- lib/ruby_llm/instrumentation/image.rb
+- lib/ruby_llm/instrumentation/moderation.rb
+- lib/ruby_llm/instrumentation/railtie.rb
+- lib/ruby_llm/instrumentation/transcription.rb
+- lib/ruby_llm/instrumentation/version.rb
+- lib/tasks/ruby_llm/instrumentation_tasks.rake
+homepage: https://github.com/sinaptia/ruby_llm-instrumentation
+licenses: []
+metadata:
+  homepage_uri: https://github.com/sinaptia/ruby_llm-instrumentation
+  source_code_uri: https://github.com/sinaptia/ruby_llm-instrumentation
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Rails instrumentation for RubyLLM
+test_files: []