activeagent 1.0.1 → 1.0.2

data/lib/active_agent/telemetry/instrumentation.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Telemetry
+    # Auto-instrumentation for ActiveAgent generation lifecycle.
+    #
+    # When included in ActiveAgent::Base, automatically traces:
+    # - Agent generation (prompt_now, generate_now)
+    # - Tool calls
+    # - Streaming events
+    # - Errors
+    #
+    # @example Enabling instrumentation
+    #   # In config/initializers/activeagent.rb
+    #   ActiveAgent::Telemetry.configure do |config|
+    #     config.enabled = true
+    #     config.endpoint = "https://api.activeagents.ai/v1/traces"
+    #     config.api_key = Rails.application.credentials.activeagents_api_key
+    #   end
+    #
+    #   # Instrumentation is automatically applied when telemetry is enabled
+    #
+    module Instrumentation
+      extend ActiveSupport::Concern
+
+      included do
+        # Hook into generation lifecycle
+        around_generate :trace_generation if respond_to?(:around_generate)
+      end
+
+      class_methods do
+        # Installs instrumentation on the agent class.
+        #
+        # Called automatically when telemetry is enabled.
+        def instrument_telemetry!
+          return if @telemetry_instrumented
+
+          prepend GenerationInstrumentation
+          @telemetry_instrumented = true
+        end
+      end
+
+      # Module prepended to intercept generation methods.
+      module GenerationInstrumentation
+        # Wraps process_prompt with telemetry tracing.
+        def process_prompt
+          return super unless Telemetry.enabled?
+
+          Telemetry.trace("#{self.class.name}.#{action_name}", span_type: :root) do |span|
+            span.set_attribute("agent.class", self.class.name)
+            span.set_attribute("agent.action", action_name.to_s)
+            span.set_attribute("agent.provider", provider_name) if respond_to?(:provider_name)
+            span.set_attribute("agent.model", model_name) if respond_to?(:model_name)
+
+            # Add prompt span
+            prompt_span = span.add_span("agent.prompt", span_type: :prompt)
+            prompt_span.set_attribute("messages.count", messages.size) if respond_to?(:messages)
+            prompt_span.finish
+
+            # Execute generation with LLM span
+            llm_span = span.add_span("llm.generate", span_type: :llm)
+            llm_span.set_attribute("llm.provider", provider_name) if respond_to?(:provider_name)
+            llm_span.set_attribute("llm.model", model_name) if respond_to?(:model_name)
+
+            begin
+              result = super
+
+              # Record token usage from response
+              if result.respond_to?(:usage) && result.usage.present?
+                usage = result.usage
+                # Usage model uses methods, not hash access
+                input_tokens = (usage.input_tokens rescue 0) || 0
+                output_tokens = (usage.output_tokens rescue 0) || 0
+                reasoning_tokens = (usage.reasoning_tokens rescue 0) || 0
+
+                llm_span.set_tokens(
+                  input: input_tokens.to_i,
+                  output: output_tokens.to_i,
+                  thinking: reasoning_tokens.to_i
+                )
+                span.set_tokens(
+                  input: input_tokens.to_i,
+                  output: output_tokens.to_i,
+                  thinking: reasoning_tokens.to_i
+                )
+              end
+
+              # Record tool calls if present
+              if result.respond_to?(:tool_calls) && result.tool_calls.present?
+                result.tool_calls.each do |tool_call|
+                  tool_span = span.add_span("tool.#{tool_call[:name]}", span_type: :tool)
+                  tool_span.set_attribute("tool.name", tool_call[:name])
+                  tool_span.set_attribute("tool.id", tool_call[:id]) if tool_call[:id]
+                  tool_span.finish
+                end
+              end
+
+              llm_span.set_status(:ok)
+              llm_span.finish
+              span.set_status(:ok)
+
+              result
+            rescue StandardError => e
+              llm_span.record_error(e)
+              llm_span.finish
+              span.record_error(e)
+              raise
+            end
+          end
+        end
+
+        # Wraps process_embed with telemetry tracing.
+        def process_embed
+          return super unless Telemetry.enabled?
+
+          Telemetry.trace("#{self.class.name}.embed", span_type: :embedding) do |span|
+            span.set_attribute("agent.class", self.class.name)
+            span.set_attribute("agent.action", "embed")
+            span.set_attribute("agent.provider", provider_name) if respond_to?(:provider_name)
+
+            begin
+              result = super
+
+              if result.respond_to?(:usage) && result.usage.present?
+                usage = result.usage
+                input_tokens = (usage.input_tokens rescue 0) || 0
+                span.set_tokens(input: input_tokens.to_i)
+              end
+
+              span.set_status(:ok)
+              result
+            rescue StandardError => e
+              span.record_error(e)
+              raise
+            end
+          end
+        end
+
+        private
+
+        def provider_name
+          self.class.generation_provider&.to_s || "unknown"
+        rescue StandardError
+          "unknown"
+        end
+
+        def model_name
+          prompt_options[:model] || "unknown"
+        rescue StandardError
+          "unknown"
+        end
+      end
+    end
+  end
+end

data/lib/active_agent/telemetry/reporter.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "uri"
+
+module ActiveAgent
+  module Telemetry
+    # Asynchronously reports traces to the telemetry endpoint.
+    #
+    # Buffers traces and sends them in batches to reduce network overhead.
+    # Uses a background thread for non-blocking transmission.
+    #
+    # @example
+    #   reporter = Reporter.new(configuration)
+    #   reporter.report(trace_payload)
+    #   reporter.flush  # Send immediately
+    #   reporter.shutdown  # Clean shutdown
+    #
+    class Reporter
+      # @return [Configuration] Telemetry configuration
+      attr_reader :configuration
+
+      def initialize(configuration)
+        @configuration = configuration
+        @buffer = []
+        @mutex = Mutex.new
+        @running = false
+        @thread = nil
+        @shutdown = false
+
+        start_flush_thread if configuration.enabled?
+      end
+
+      # Adds a trace to the buffer for transmission.
+      #
+      # @param trace [Hash] Trace payload
+      # @return [void]
+      def report(trace)
+        return unless configuration.enabled?
+
+        @mutex.synchronize do
+          @buffer << trace
+
+          # Flush immediately if buffer is full
+          if @buffer.size >= configuration.batch_size
+            flush_buffer
+          end
+        end
+      end
+
+      # Flushes all buffered traces immediately.
+      #
+      # @return [void]
+      def flush
+        @mutex.synchronize do
+          flush_buffer
+        end
+      end
+
+      # Shuts down the reporter, flushing remaining traces.
+      #
+      # @return [void]
+      def shutdown
+        @shutdown = true
+        flush
+        @thread&.join(5) # Wait up to 5 seconds for thread to finish
+      end
+
+      private
+
+      # Starts the background flush thread.
+      def start_flush_thread
+        @running = true
+        @thread = Thread.new do
+          Thread.current.name = "activeagent-telemetry-reporter"
+
+          while @running && !@shutdown
+            sleep(configuration.flush_interval)
+
+            @mutex.synchronize do
+              flush_buffer if @buffer.any?
+            end
+          end
+        end
+      end
+
+      # Flushes the buffer by sending traces to the endpoint.
+      #
+      # Must be called within @mutex synchronization.
+      def flush_buffer
+        return if @buffer.empty?
+
+        traces = @buffer.dup
+        @buffer.clear
+
+        Thread.new { send_traces(traces) }
+      end
+
+      # Sends traces to the configured endpoint.
+      #
+      # @param traces [Array<Hash>] Traces to send
+      def send_traces(traces)
+        # Use direct database storage for local mode
+        if configuration.local_storage?
+          store_traces_locally(traces)
+          return
+        end
+
+        uri = URI.parse(configuration.endpoint)
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = configuration.timeout
+        http.read_timeout = configuration.timeout
+
+        request = Net::HTTP::Post.new(uri.request_uri)
+        request["Content-Type"] = "application/json"
+        request["Authorization"] = "Bearer #{configuration.api_key}"
+        request["User-Agent"] = "ActiveAgent/#{ActiveAgent::VERSION} Ruby/#{RUBY_VERSION}"
+        request["X-Service-Name"] = configuration.resolved_service_name
+        request["X-Environment"] = configuration.environment
+
+        payload = {
+          traces: traces,
+          sdk: {
+            name: "activeagent",
+            version: ActiveAgent::VERSION,
+            language: "ruby",
+            runtime_version: RUBY_VERSION
+          }
+        }
+
+        request.body = JSON.generate(payload)
+
+        response = http.request(request)
+
+        unless response.is_a?(Net::HTTPSuccess)
+          log_error("Failed to send traces: #{response.code} #{response.message}")
+        end
+      rescue StandardError => e
+        log_error("Error sending traces: #{e.class} - #{e.message}")
+      end
+
+      # Stores traces directly in the local database.
+      #
+      # @param traces [Array<Hash>] Traces to store
+      def store_traces_locally(traces)
+        sdk_info = {
+          name: "activeagent",
+          version: ActiveAgent::VERSION,
+          language: "ruby",
+          runtime_version: RUBY_VERSION
+        }
+
+        traces.each do |trace|
+          # Skip if trace already exists (idempotency)
+          next if ActiveAgent::TelemetryTrace.exists?(trace_id: trace["trace_id"])
+
+          ActiveAgent::TelemetryTrace.create_from_payload(trace, sdk_info)
+        rescue StandardError => e
+          log_error("Failed to store trace locally: #{e.class} - #{e.message}")
+        end
+      rescue StandardError => e
+        log_error("Error storing traces locally: #{e.class} - #{e.message}")
+      end
+
+      # Logs an error message.
+      #
+      # @param message [String] Error message
+      def log_error(message)
+        configuration.resolved_logger.error("[ActiveAgent::Telemetry] #{message}")
+      end
+    end
+  end
+end

data/lib/active_agent/telemetry/span.rb

@@ -0,0 +1,267 @@
+# frozen_string_literal: true
+
+module ActiveAgent
+  module Telemetry
+    # Represents a single span in a trace.
+    #
+    # Spans capture discrete operations within a trace, such as LLM calls,
+    # tool invocations, or prompt rendering. Each span has timing, attributes,
+    # and can have child spans.
+    #
+    # @example Creating a span
+    #   span = Span.new("llm.generate", trace_id: trace.trace_id)
+    #   span.set_attribute("provider", "anthropic")
+    #   span.set_attribute("model", "claude-3-5-sonnet")
+    #   span.set_tokens(input: 100, output: 50)
+    #   span.finish
+    #
+    class Span
+      # Span types for categorization
+      TYPES = {
+        root: "root",           # Root span for entire generation
+        prompt: "prompt",       # Prompt preparation/rendering
+        llm: "llm",             # LLM API call
+        tool: "tool",           # Tool invocation
+        thinking: "thinking",   # Extended thinking (Anthropic)
+        embedding: "embedding", # Embedding generation
+        error: "error"          # Error handling
+      }.freeze
+
+      # Span status codes
+      STATUS = {
+        unset: "UNSET",
+        ok: "OK",
+        error: "ERROR"
+      }.freeze
+
+      # @return [String] Unique identifier for this span
+      attr_reader :span_id
+
+      # @return [String] Trace ID this span belongs to
+      attr_reader :trace_id
+
+      # @return [String, nil] Parent span ID
+      attr_reader :parent_span_id
+
+      # @return [String] Span name (e.g., "llm.generate", "tool.get_weather")
+      attr_reader :name
+
+      # @return [String] Span type from TYPES
+      attr_reader :span_type
+
+      # @return [Time] When the span started
+      attr_reader :start_time
+
+      # @return [Time, nil] When the span ended
+      attr_reader :end_time
+
+      # @return [Hash] Span attributes
+      attr_reader :attributes
+
+      # @return [Array<Span>] Child spans
+      attr_reader :children
+
+      # @return [String] Status code from STATUS
+      attr_reader :status
+
+      # @return [String, nil] Status message
+      attr_reader :status_message
+
+      # @return [Array<Hash>] Events recorded during the span
+      attr_reader :events
+
+      # Creates a new span.
+      #
+      # @param name [String] Span name
+      # @param trace_id [String] Parent trace ID
+      # @param parent_span_id [String, nil] Parent span ID
+      # @param span_type [Symbol] Type of span
+      # @param attributes [Hash] Initial attributes
+      def initialize(name, trace_id:, parent_span_id: nil, span_type: :root, **attributes)
+        @span_id = SecureRandom.hex(8)
+        @trace_id = trace_id
+        @parent_span_id = parent_span_id
+        @name = name
+        @span_type = TYPES[span_type] || span_type.to_s
+        @start_time = Time.current
+        @end_time = nil
+        @attributes = attributes.transform_keys(&:to_s)
+        @children = []
+        @status = STATUS[:unset]
+        @status_message = nil
+        @events = []
+        @tokens = { input: 0, output: 0, thinking: 0, total: 0 }
+      end
+
+      # Creates a child span.
+      #
+      # @param name [String] Child span name
+      # @param span_type [Symbol] Type of span
+      # @param attributes [Hash] Span attributes
+      # @return [Span] The child span
+      def add_span(name, span_type: :root, **attributes)
+        child = Span.new(
+          name,
+          trace_id: trace_id,
+          parent_span_id: span_id,
+          span_type: span_type,
+          **attributes
+        )
+        @children << child
+        child
+      end
+
+      # Sets a single attribute.
+      #
+      # @param key [String, Symbol] Attribute key
+      # @param value [Object] Attribute value
+      # @return [self]
+      def set_attribute(key, value)
+        @attributes[key.to_s] = value
+        self
+      end
+
+      # Sets multiple attributes at once.
+      #
+      # @param attrs [Hash] Attributes to set
+      # @return [self]
+      def set_attributes(attrs)
+        attrs.each { |k, v| set_attribute(k, v) }
+        self
+      end
+
+      # Sets token usage for LLM spans.
+      #
+      # @param input [Integer] Input token count
+      # @param output [Integer] Output token count
+      # @param thinking [Integer] Thinking token count (Anthropic extended thinking)
+      # @return [self]
+      def set_tokens(input: 0, output: 0, thinking: 0)
+        @tokens = {
+          input: input,
+          output: output,
+          thinking: thinking,
+          total: input + output + thinking
+        }
+        set_attribute("tokens.input", input)
+        set_attribute("tokens.output", output)
+        set_attribute("tokens.thinking", thinking) if thinking > 0
+        set_attribute("tokens.total", @tokens[:total])
+        self
+      end
+
+      # Returns token usage.
+      #
+      # @return [Hash] Token counts
+      def tokens
+        @tokens.dup
+      end
+
+      # Sets the span status.
+      #
+      # @param code [Symbol] Status code (:ok, :error, :unset)
+      # @param message [String, nil] Optional status message
+      # @return [self]
+      def set_status(code, message = nil)
+        @status = STATUS[code] || STATUS[:unset]
+        @status_message = message
+        self
+      end
+
+      # Records an error on the span.
+      #
+      # @param error [Exception] The error to record
+      # @return [self]
+      def record_error(error)
+        set_status(:error, error.message)
+        set_attribute("error.type", error.class.name)
+        set_attribute("error.message", error.message)
+        set_attribute("error.backtrace", error.backtrace&.first(10)&.join("\n"))
+
+        add_event("exception", {
+          "exception.type" => error.class.name,
+          "exception.message" => error.message,
+          "exception.stacktrace" => error.backtrace&.join("\n")
+        })
+
+        self
+      end
+
+      # Adds an event to the span.
+      #
+      # @param name [String] Event name
+      # @param attributes [Hash] Event attributes
+      # @return [self]
+      def add_event(name, attributes = {})
+        @events << {
+          name: name,
+          timestamp: Time.current.iso8601(6),
+          attributes: attributes.transform_keys(&:to_s)
+        }
+        self
+      end
+
+      # Marks the span as finished.
+      #
+      # @return [self]
+      def finish
+        @end_time = Time.current
+        set_status(:ok) if @status == STATUS[:unset]
+        self
+      end
+
+      # Returns whether the span is finished.
+      #
+      # @return [Boolean]
+      def finished?
+        !@end_time.nil?
+      end
+
+      # Returns the duration in milliseconds.
+      #
+      # @return [Float, nil] Duration or nil if not finished
+      def duration_ms
+        return nil unless finished?
+
+        ((@end_time - @start_time) * 1000).round(2)
+      end
+
+      # Serializes the span for transmission.
+      #
+      # @return [Hash] Serialized span data
+      def to_h
+        {
+          span_id: span_id,
+          trace_id: trace_id,
+          parent_span_id: parent_span_id,
+          name: name,
+          type: span_type,
+          start_time: start_time.iso8601(6),
+          end_time: end_time&.iso8601(6),
+          duration_ms: duration_ms,
+          status: status,
+          status_message: status_message,
+          attributes: attributes,
+          tokens: tokens,
+          events: events,
+          children: children.map(&:to_h)
+        }
+      end
+
+      # Executes a block and records timing/errors.
+      #
+      # @yield Block to execute within the span
+      # @return [Object] Result of the block
+      def measure
+        result = yield
+        set_status(:ok)
+        result
+      rescue StandardError => e
+        record_error(e)
+        raise
+      ensure
+        finish
+      end
+    end
+  end
+end