ask-instrumentation 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: dcb0532585201d8c5d321aa173794fc0774b34e634b0b65ca0dcee15f2819649
+  data.tar.gz: a56ac64e426a4a01c27983cb4e705eba13ccb4a71b075ace4a2595cf4ca63229
+SHA512:
+  metadata.gz: 5aa9ae01411b237e901041d55f604283dea028e2262a5f2c818317d2e57e490a0f065f9462a5efee089b6b1e900960ac9569adc553022356d5821eea8d9d0d72
+  data.tar.gz: 69d831ccbbd59ee50374ede18ef77a073fb856b1089e8676ba18cf284bb853718fcf34d92b4c5e0f672d93f3ef20d536e1f4b3d1678834f4de354ca90251a214

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+# Changelog
+
+## [0.1.0] - Unreleased
+
+### Added
+- Initial release

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Kaka Ruto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,277 @@
+# ask-instrumentation
+
+[![Gem Version](https://badge.fury.io/rb/ask-instrumentation.svg)](https://badge.fury.io/rb/ask-instrumentation)
+[![CI](https://github.com/ask-rb/ask-instrumentation/actions/workflows/ci.yml/badge.svg)](https://github.com/ask-rb/ask-instrumentation/actions/workflows/ci.yml)
+
+LLM observability for the ask-rb ecosystem. Emits `ActiveSupport::Notifications`
+events for chat completions, embeddings, tool calls, and image generation.
+
+Works with **any** LLM provider — not tied to a specific one. Subscribe to events
+for cost tracking, logging, analytics, or alerting.
+
+## Installation
+
+```ruby
+gem "ask-instrumentation"
+```
+
+Or install it yourself:
+
+```bash
+gem install ask-instrumentation
+```
+
+## Quick Start
+
+```ruby
+require "ask/instrumentation"
+
+# Subscribe to all ask events
+Ask::Instrumentation.subscribe do |event|
+  puts "#{event.name}: #{event.duration}ms"
+end
+
+# Instrument a chat completion
+Ask::Instrumentation.instrument("chat.ask", provider: "openai", model: "gpt-4") do
+  # your LLM call here
+end
+
+# Wrap with metadata context
+Ask::Instrumentation.with_metadata(user_id: 42, session_id: "abc") do
+  Ask::Instrumentation.instrument("chat.ask", { provider: "openai", model: "gpt-4" }) do
+    # ...
+  end
+end
+```
+
+## Events
+
+| Event | Payload | Description |
+|---|---|---|
+| `chat.ask` | provider, model, input_tokens, output_tokens, duration | Chat completion |
+| `chat.stream.ask` | provider, model, input_tokens, output_tokens, duration | Streaming chat |
+| `tool.ask` | provider, tool_name, tool_args, duration | Tool call |
+| `embedding.ask` | provider, model, input_tokens, duration | Embedding |
+| `image.ask` | provider, model, size, duration | Image generation |
+
+## Examples
+
+### Cost Tracking
+
+Subscribe to chat events and sum model costs:
+
+```ruby
+require "ask/instrumentation"
+
+COST_PER_TOKEN = {
+  "gpt-4"        => { input: 0.03 / 1000, output: 0.06 / 1000 },
+  "gpt-3.5-turbo" => { input: 0.001 / 1000, output: 0.002 / 1000 },
+  "claude-3-opus" => { input: 0.015 / 1000, output: 0.075 / 1000 }
+}.freeze
+
+total_cost = 0.0
+
+Ask::Instrumentation.subscribe(/chat\.ask/) do |event|
+  payload = event.payload
+  model   = payload[:model]
+  pricing = COST_PER_TOKEN[model]
+  next unless pricing
+
+  cost = (payload[:input_tokens].to_i * pricing[:input]) +
+         (payload[:output_tokens].to_i * pricing[:output])
+  total_cost += cost
+
+  puts "[COST] #{model}: $%.6f (total: $%.4f)" % [cost, total_cost]
+end
+
+# Later, in your application code:
+Ask::Instrumentation.with_metadata(session_id: "sess_123") do
+  Ask::Instrumentation.instrument("chat.ask",
+    provider: "openai",
+    model: "gpt-4",
+    input_tokens: 150,
+    output_tokens: 50
+  ) do
+    # actual LLM API call
+  end
+end
+```
+
+### Request Logging
+
+Log all LLM events to a file with structured data:
+
+```ruby
+require "ask/instrumentation"
+require "logger"
+
+logger = Logger.new("log/llm.log")
+
+Ask::Instrumentation.subscribe do |event|
+  logger.info({
+    event:    event.name,
+    duration: event.duration.round(2),
+    **event.payload
+  }.to_json)
+end
+```
+
+### Usage Analytics with Metadata
+
+Track per-user and per-session usage:
+
+```ruby
+require "ask/instrumentation"
+
+# In your application (e.g., a Rails controller):
+class ChatController < ApplicationController
+  def create
+    Ask::Instrumentation.with_metadata(
+      user_id:    current_user.id,
+      session_id: request.session.id,
+      request_id: request.request_id
+    ) do
+      # All events emitted here automatically include user/session context
+      response = llm_client.chat(params[:message])
+      render json: response
+    end
+  end
+end
+
+# In a monitoring background worker:
+Ask::Instrumentation.subscribe(/chat\.ask/) do |event|
+  payload = event.payload
+  UsageReport.increment(
+    user_id:    payload[:user_id],
+    model:      payload[:model],
+    tokens_in:  payload[:input_tokens],
+    tokens_out: payload[:output_tokens]
+  )
+end
+```
+
+### Integration with ask-llm-providers
+
+When using the ask-rb ecosystem, providers emit events automatically:
+
+```ruby
+require "ask/provider"
+
+# Events are emitted automatically — no manual instrumentation needed.
+provider = Ask::Provider.for(:openai)
+provider.complete(prompt: "Hello!") # emits chat.ask
+
+# Subscribe to track everything:
+Ask::Instrumentation.subscribe do |event|
+  puts "[#{event.name}] #{event.payload[:model]} (#{event.duration}ms)"
+end
+```
+
+### With Your Own Provider
+
+You can emit events from any custom provider:
+
+```ruby
+class MyCustomProvider
+  def complete(prompt)
+    Ask::Instrumentation.instrument("chat.ask",
+      provider: "my_custom",
+      model: "my-model-v1",
+      input_tokens: prompt.length / 4
+    ) do
+      response = call_api(prompt)
+      # enrich payload after the call by returning a hash from the block?
+      # No — use with_metadata or include everything upfront.
+      response
+    end
+  end
+end
+```
+
+## API
+
+### `.subscribe(pattern = /\.ask$/, &block)`
+
+Subscribe to ask events. Accepts an optional pattern (defaults to all `.ask` events).
+
+```ruby
+# All ask events
+Ask::Instrumentation.subscribe { |event| ... }
+
+# Only chat events
+Ask::Instrumentation.subscribe(/chat\.ask/) { |event| ... }
+```
+
+### `.unsubscribe(subscriber_or_name)`
+
+Remove a subscriber by passing the object returned from `subscribe` or a string/regexp.
+
+```ruby
+subscriber = Ask::Instrumentation.subscribe { |e| ... }
+Ask::Instrumentation.unsubscribe(subscriber)
+```
+
+### `.instrument(name, payload = {}, &block)`
+
+Emit an event. The block is instrumented and its return value is passed through.
+Metadata from `with_metadata` is automatically merged into the payload.
+
+```ruby
+result = Ask::Instrumentation.instrument("chat.ask",
+  provider: "openai",
+  model: "gpt-4"
+) do
+  llm_call
+end
+```
+
+### `.with_metadata(hash, &block)`
+
+Set thread-local metadata that is merged into all events emitted inside the block.
+Nested blocks merge inner metadata into outer, with inner values taking precedence.
+
+```ruby
+Ask::Instrumentation.with_metadata(user_id: 42) do
+  Ask::Instrumentation.instrument("chat.ask", provider: "openai", model: "gpt-4") do
+    # event payload includes user_id: 42
+  end
+end
+```
+
+### `.current_metadata`
+
+Return the current thread's metadata hash (empty hash if no metadata is set).
+
+```ruby
+Ask::Instrumentation.current_metadata # => {}
+```
+
+## Thread Safety
+
+All metadata is stored in `Thread.current`, making it safe to use in concurrent
+environments. Each thread has its own metadata context:
+
+```ruby
+Ask::Instrumentation.with_metadata(thread: "main") do
+  Thread.new do
+    # This thread has its own metadata context
+    Ask::Instrumentation.current_metadata # => {}
+    Ask::Instrumentation.with_metadata(thread: "worker") do
+      # ...
+    end
+  end.join
+end
+```
+
+## Development
+
+```bash
+git clone https://github.com/ask-rb/ask-instrumentation.git
+cd ask-instrumentation
+bundle install
+bundle exec rake test
+```
+
+## License
+
+MIT

data/lib/ask/instrumentation/chat.rb

@@ -0,0 +1,13 @@
+module Ask
+  module Instrumentation
+    # Placeholder for Chat-specific instrumentation helpers.
+    #
+    # This module will be loaded automatically when
+    # +Ask::Instrumentation::Chat+ is referenced (via +autoload+).
+    #
+    # Future versions may include helper methods for instrumenting
+    # streaming chat completions.
+    module Chat
+    end
+  end
+end

data/lib/ask/instrumentation/embedding.rb

@@ -0,0 +1,10 @@
+module Ask
+  module Instrumentation
+    # Placeholder for Embedding-specific instrumentation helpers.
+    #
+    # This module will be loaded automatically when
+    # +Ask::Instrumentation::Embedding+ is referenced (via +autoload+).
+    module Embedding
+    end
+  end
+end

data/lib/ask/instrumentation/tool.rb

@@ -0,0 +1,10 @@
+module Ask
+  module Instrumentation
+    # Placeholder for Tool-specific instrumentation helpers.
+    #
+    # This module will be loaded automatically when
+    # +Ask::Instrumentation::Tool+ is referenced (via +autoload+).
+    module Tool
+    end
+  end
+end

data/lib/ask/instrumentation/version.rb

@@ -0,0 +1,5 @@
+module Ask
+  module Instrumentation
+    VERSION = "0.1.0"
+  end
+end

data/lib/ask/instrumentation.rb

@@ -0,0 +1,163 @@
+require "active_support"
+require "active_support/notifications"
+require_relative "instrumentation/version"
+
+module Ask
+  # Instrumentation for LLM observability in the ask-rb ecosystem.
+  #
+  # Provides a thin wrapper around +ActiveSupport::Notifications+ for emitting
+  # and subscribing to LLM events such as chat completions, embeddings, tool
+  # calls, and image generation.
+  #
+  # Events follow the +{operation}.ask+ naming convention:
+  #
+  #   chat.ask        # Chat completion
+  #   chat.stream.ask # Streaming chat
+  #   tool.ask        # Tool execution
+  #   embedding.ask   # Embedding generation
+  #   image.ask       # Image generation
+  #
+  # Each event payload includes provider-agnostic keys such as +provider+,
+  # +model+, +duration+, and provider-specific metrics (+input_tokens+,
+  # +output_tokens+, etc.).
+  #
+  # == Usage
+  #
+  #   require "ask/instrumentation"
+  #
+  #   # Subscribe to all ask events
+  #   Ask::Instrumentation.subscribe do |event|
+  #     puts "#{event.name}: #{event.duration}ms"
+  #   end
+  #
+  #   # Subscribe with a callable subscriber
+  #   Ask::Instrumentation.subscribe(MySubscriber.new)
+  #
+  #   # Instrument a chat completion
+  #   Ask::Instrumentation.instrument("chat.ask", provider: "openai", model: "gpt-4") do
+  #     # your LLM call here
+  #   end
+  #
+  #   # Wrap with metadata context
+  #   Ask::Instrumentation.with_metadata(user_id: 42, session_id: "abc") do
+  #     Ask::Instrumentation.instrument("chat.ask", provider: "openai", model: "gpt-4") do
+  #       # ...
+  #     end
+  #   end
+  module Instrumentation
+    autoload :Chat, "ask/instrumentation/chat"
+    autoload :Embedding, "ask/instrumentation/embedding"
+    autoload :Tool, "ask/instrumentation/tool"
+
+    class << self
+      # Subscribe to ask instrumentation events.
+      #
+      # Accepts a callable subscriber (an object that responds to +#call+)
+      # and/or a block. When a pattern is given, only events matching the
+      # pattern are delivered to the subscriber.
+      #
+      # @overload subscribe(pattern = /\.ask$/, subscriber)
+      #   @param pattern [Regexp, String] Optional event name pattern
+      #   @param subscriber [#call] An object that responds to +#call(event)+
+      #   @return [ActiveSupport::Notifications::Fanout::Subscriber]
+      #
+      # @overload subscribe(pattern = /\.ask$/, &block)
+      #   @param pattern [Regexp, String] Optional event name pattern
+      #   @param block [Proc] The callback invoked on each matching event
+      #   @return [ActiveSupport::Notifications::Fanout::Subscriber]
+      #
+      # @example Subscribe with a block
+      #   Ask::Instrumentation.subscribe { |event| puts event.name }
+      #
+      # @example Subscribe with a callable object
+      #   Ask::Instrumentation.subscribe(MySubscriber.new)
+      #
+      # @example Subscribe with a pattern and callable
+      #   Ask::Instrumentation.subscribe(/chat\.ask/, MySubscriber.new)
+      def subscribe(pattern = /\.ask$/, subscriber = nil, &block)
+        ActiveSupport::Notifications.subscribe(pattern, subscriber || block)
+      end
+
+      # Unsubscribe from events.
+      #
+      # Delegates to +ActiveSupport::Notifications.unsubscribe+. Accepts either
+      # the subscriber object returned from {subscribe} or a string/regexp
+      # pattern.
+      #
+      # @param subscriber_or_name [ActiveSupport::Notifications::Fanout::Subscriber, String, Regexp]
+      #   The subscriber to remove
+      # @return [void]
+      #
+      # @example
+      #   subscriber = Ask::Instrumentation.subscribe { |e| puts e.name }
+      #   Ask::Instrumentation.unsubscribe(subscriber)
+      def unsubscribe(subscriber_or_name)
+        ActiveSupport::Notifications.unsubscribe(subscriber_or_name)
+      end
+
+      # Instrument a block of code, wrapping it in an +ActiveSupport::Notifications+ event.
+      #
+      # The event name should follow the +{operation}.ask+ convention.
+      # The payload is automatically merged with the current thread metadata
+      # (set via {with_metadata}) before being emitted.
+      #
+      # @param name [String] The event name (e.g. +"chat.ask"+)
+      # @param payload [Hash] The event payload
+      # @yield The block to instrument
+      # @return [Object] The return value of the block
+      #
+      # @example
+      #   Ask::Instrumentation.instrument("chat.ask", provider: "openai", model: "gpt-4") do
+      #     # LLM call
+      #   end
+      def instrument(name, payload = {})
+        merged = current_metadata.merge(payload)
+        ActiveSupport::Notifications.instrument(name, merged) { yield if block_given? }
+      end
+
+      # Execute a block with thread-local metadata that gets merged into all
+      # events emitted within the block.
+      #
+      # Nested calls merge the inner metadata into the outer, with the inner
+      # values taking precedence for duplicate keys. The outer metadata is
+      # restored after the inner block completes.
+      #
+      # Metadata is useful for attaching context such as +user_id+,
+      # +session_id+, +request_id+, or +trace_id+ to every event.
+      #
+      # @param metadata [Hash] The metadata hash to attach
+      # @yield The block to execute with the metadata context
+      # @return [Object] The return value of the block
+      #
+      # @example
+      #   Ask::Instrumentation.with_metadata(user_id: 42) do
+      #     Ask::Instrumentation.instrument("chat.ask", provider: "openai", model: "gpt-4") do
+      #       # ... event payload will include user_id: 42
+      #     end
+      #   end
+      def with_metadata(metadata = {})
+        previous = Thread.current[:ask_instrumentation_metadata]
+        Thread.current[:ask_instrumentation_metadata] = (previous || {}).merge(metadata)
+        yield
+      ensure
+        Thread.current[:ask_instrumentation_metadata] = previous
+      end
+
+      # Return the current thread's metadata hash.
+      #
+      # This is the metadata hash that will be merged into any event payload
+      # emitted via {instrument} in the current thread.
+      #
+      # @return [Hash] The current metadata (empty hash if none set)
+      #
+      # @example
+      #   Ask::Instrumentation.current_metadata # => {}
+      #   Ask::Instrumentation.with_metadata(user_id: 42) do
+      #     Ask::Instrumentation.current_metadata # => { user_id: 42 }
+      #   end
+      def current_metadata
+        Thread.current[:ask_instrumentation_metadata] || {}
+      end
+    end
+  end
+end

data/lib/ask-instrumentation.rb

@@ -0,0 +1 @@
+require_relative "ask/instrumentation"

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: ask-instrumentation
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Kaka Ruto
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '7.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.25'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.25'
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+description: Emits ActiveSupport::Notifications events for chat completions, embeddings,
+  tool calls, and image generation. Works with any LLM provider. Subscribe to events
+  for cost tracking, logging, analytics, or alerting.
+email:
+- kaka@myrrlabs.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- lib/ask-instrumentation.rb
+- lib/ask/instrumentation.rb
+- lib/ask/instrumentation/chat.rb
+- lib/ask/instrumentation/embedding.rb
+- lib/ask/instrumentation/tool.rb
+- lib/ask/instrumentation/version.rb
+homepage: https://github.com/ask-rb/ask-instrumentation
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/ask-rb/ask-instrumentation
+  source_code_uri: https://github.com/ask-rb/ask-instrumentation
+  changelog_uri: https://github.com/ask-rb/ask-instrumentation/blob/master/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.3
+specification_version: 4
+summary: LLM observability for the ask-rb ecosystem
+test_files: []