RubyGems - coolhand - Versions diffs - 0.2.0 → 0.4.0 - Mend

coolhand 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (32) hide show

checksums.yaml +4 -4
data/.rubocop.yml +12 -0
data/CHANGELOG.md +68 -1
data/CLAUDE.md +34 -0
data/README.md +152 -28
data/docs/anthropic.md +11 -11
data/docs/elevenlabs.md +6 -4
data/lib/coolhand/api_service.rb +277 -0
data/lib/coolhand/base_interceptor.rb +213 -0
data/lib/coolhand/collector.rb +19 -0
data/lib/coolhand/configuration.rb +79 -0
data/lib/coolhand/default_exclude_api_patterns.yml +9 -0
data/lib/coolhand/default_intercept_addresses.yml +19 -0
data/lib/coolhand/feedback_service.rb +15 -0
data/lib/coolhand/logger_service.rb +112 -0
data/lib/coolhand/net_http_interceptor.rb +163 -0
data/lib/coolhand/open_ai/batch_result_processor.rb +139 -0
data/lib/coolhand/open_ai/webhook_validator.rb +127 -0
data/lib/coolhand/{ruby/version.rb → version.rb} +1 -3
data/lib/coolhand/vertex/batch_result_processor.rb +84 -0
data/lib/coolhand/webhook_interceptor.rb +39 -0
data/lib/coolhand.rb +113 -2
metadata +34 -18
data/lib/coolhand/ruby/anthropic_interceptor.rb +0 -300
data/lib/coolhand/ruby/api_service.rb +0 -226
data/lib/coolhand/ruby/base_interceptor.rb +0 -148
data/lib/coolhand/ruby/collector.rb +0 -21
data/lib/coolhand/ruby/configuration.rb +0 -38
data/lib/coolhand/ruby/faraday_interceptor.rb +0 -129
data/lib/coolhand/ruby/feedback_service.rb +0 -17
data/lib/coolhand/ruby/logger_service.rb +0 -114
data/lib/coolhand/ruby.rb +0 -121

data/lib/coolhand/api_service.rb ADDED Viewed

@@ -0,0 +1,277 @@
+# frozen_string_literal: true
+require "net/http"
+require "uri"
+require "json"
+require_relative "collector"
+module Coolhand
+  class ApiService
+    attr_reader :api_endpoint
+    def initialize(endpoint = "v2/llm_request_logs")
+      @api_endpoint = "#{base_url}/#{endpoint}"
+    end
+    def send_llm_request_log(request_data)
+      payload = {
+        llm_request_log: request_data.merge(
+          collector: Collector.get_collector_string
+        )
+      }
+      if debug_mode?
+        log_separator
+        log "🛠️ Debug Mode - Request payload prepared but not sent to API:"
+        log JSON.pretty_generate(sanitize_payload_for_json(payload))
+        nil
+      else
+        send_request(payload, "✅ Successfully sent request metadata")
+      end
+    end
+    def configuration
+      Coolhand.configuration
+    end
+    def base_url
+      configuration.base_url
+    end
+    def api_key
+      configuration.api_key
+    end
+    def silent
+      configuration.silent
+    end
+    def debug_mode?
+      configuration.debug_mode
+    end
+    protected
+    # Add collector field to the data being sent
+    def add_collector_to_data(data, collection_method = nil)
+      data.merge(collector: Collector.get_collector_string(collection_method))
+    end
+    def create_request_options(_payload)
+      {
+        "Content-Type" => "application/json",
+        "X-API-Key" => api_key
+      }
+    end
+    def send_request(payload, success_message)
+      return nil if missing_api_key?
+      uri = URI.parse(@api_endpoint)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = (uri.scheme == "https")
+      request = Net::HTTP::Post.new(uri.request_uri)
+      headers = create_request_options(payload)
+      headers.each do |key, value|
+        # Ensure header values are UTF-8 encoded
+        encoded_value = value.is_a?(String) ? value.dup.force_encoding("UTF-8") : value
+        request[key] = encoded_value
+      end
+      # Clean payload and ensure UTF-8 encoding before JSON generation
+      cleaned_payload = sanitize_payload_for_json(payload)
+      json_body = JSON.generate(cleaned_payload)
+      # Ensure the request body is properly encoded as UTF-8
+      request.body = json_body.force_encoding("UTF-8")
+      begin
+        response = http.request(request)
+        if response.is_a?(Net::HTTPSuccess)
+          result = JSON.parse(response.body, symbolize_names: true)
+          log success_message
+          result
+        else
+          body = response.body.force_encoding("UTF-8") if response.body
+          # Only show first part of HTML error pages
+          error_msg = if body&.include?("<!DOCTYPE html>")
+            "#{body[0..200]}... [HTML error page truncated]"
+          else
+            body
+          end
+          log "❌ Request failed: #{response.code} - #{error_msg}"
+          nil
+        end
+      rescue StandardError => e
+        log "❌ Request error: #{e.message}"
+        nil
+      end
+    end
+    def log(*args)
+      puts args.join(" ") unless silent
+    end
+    def log_separator
+      log("═" * 60) unless silent
+    end
+    def create_feedback(feedback, collection_method = nil)
+      return nil if !debug_mode? && missing_api_key?
+      normalized = normalize_feedback_sentiment(feedback)
+      feedback_with_collector = add_collector_to_data(normalized, collection_method)
+      payload = {
+        llm_request_log_feedback: feedback_with_collector
+      }
+      log_feedback_info(normalized)
+      if debug_mode?
+        log_separator
+        log "🛠️ Debug Mode - Request payload prepared but not sent to API:"
+        log JSON.pretty_generate(payload)
+        nil
+      else
+        result = send_request(
+          payload,
+          "✅ Successfully created feedback with ID: #{feedback[:llm_request_log_id] || 'N/A'}"
+        )
+        log_separator
+        result
+      end
+    end
+    def create_log(captured_data, collection_method = nil)
+      return nil if !debug_mode? && missing_api_key?
+      raw_request_with_collector = add_collector_to_data({ raw_request: captured_data }, collection_method)
+      payload = {
+        llm_request_log: raw_request_with_collector
+      }
+      log_request_info(captured_data)
+      if debug_mode?
+        log_separator
+        log "🛠️ Debug Mode - Request payload prepared but not sent to API:"
+        log JSON.pretty_generate(payload)
+        nil
+      else
+        result = send_request(
+          payload,
+          "✅ Successfully logged to API"
+        )
+        puts "✅ Successfully logged to API with ID: #{result[:id]}" if result && !silent
+        log_separator
+        result
+      end
+    end
+    # Filter list of known binary/problematic field names by service
+    BINARY_DATA_FILTERS = {
+      # ElevenLabs fields that contain binary audio data
+      elevenlabs: %w[
+        full_audio
+        audio
+        audio_data
+        raw_audio
+        audio_base64
+        voice_sample
+        audio_url
+      ],
+      # OpenAI fields that might contain binary data
+      openai: %w[
+        file_content
+        audio_data
+        image_data
+        binary_content
+      ]
+    }.freeze
+    private
+    def missing_api_key?
+      return false if Coolhand.required_field?(api_key)
+      Coolhand.log "⚠️  Coolhand: API key is missing — skipping log for this request."
+      true
+    end
+    # Get all filtered field names as a flat array
+    def filtered_field_names
+      @filtered_field_names ||= BINARY_DATA_FILTERS.values.flatten.map(&:downcase)
+    end
+    # Recursively sanitize payload to remove known problematic fields
+    def sanitize_payload_for_json(obj)
+      case obj
+      when Hash
+        obj.each_with_object({}) do |(key, value), sanitized|
+          key_str = key.to_s.downcase
+          # Skip if key matches any filtered field name
+          next if filtered_field_names.any? { |filter| key_str.include?(filter) }
+          sanitized[key] = sanitize_payload_for_json(value)
+        end
+      when Array
+        obj.map { |item| sanitize_payload_for_json(item) }
+      else
+        obj
+      end
+    rescue StandardError => e
+      log "⚠️ Warning: Error sanitizing payload: #{e.message}"
+      obj
+    end
+    def normalize_feedback_sentiment(feedback)
+      return feedback.except(:like) if feedback[:sentiment]
+      return feedback if feedback[:like].nil?
+      sentiment = feedback[:like] ? "like" : "dislike"
+      feedback.except(:like).merge(sentiment: sentiment)
+    end
+    def log_feedback_info(feedback)
+      return if silent
+      # Log the appropriate identifier based on what was provided
+      if feedback[:llm_request_log_id]
+        puts "\n📝 CREATING FEEDBACK for LLM Request Log ID: #{feedback[:llm_request_log_id]}"
+      elsif feedback[:llm_provider_unique_id]
+        puts "\n📝 CREATING FEEDBACK for Provider Unique ID: #{feedback[:llm_provider_unique_id]}"
+      else
+        puts "\n📝 CREATING FEEDBACK"
+      end
+      puts "💬 Sentiment: #{feedback[:sentiment]}" if feedback[:sentiment]
+      puts "🔗 Workload: #{feedback[:workload_hashid]}" if feedback[:workload_hashid]
+      if feedback[:explanation]
+        explanation = feedback[:explanation]
+        truncated = explanation.length > 100 ? "#{explanation[0..99]}..." : explanation
+        puts "💭 Explanation: #{truncated}"
+      end
+      puts "📤 Sending to: #{@api_endpoint}"
+    end
+    def log_request_info(captured_data)
+      return if silent
+      puts "\n🎉 LOGGING OpenAI API Call #{@api_endpoint}"
+      puts captured_data
+      puts "📤 Sending to: #{@api_endpoint}"
+    end
+  end
+end

data/lib/coolhand/base_interceptor.rb ADDED Viewed

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+module Coolhand
+  # Base module with common functionality for all interceptors
+  module BaseInterceptor
+    module_function
+    def extract_response_data(response)
+      case response
+      when Hash
+        response
+      when Struct
+        response.to_h
+      else
+        # Handle streaming responses - these are often enumerator objects
+        # that can't be serialized directly
+        if response.class.name.include?("Stream") || response.respond_to?(:each)
+          {
+            response_type: "streaming",
+            class: response.class.name,
+            note: "Streaming response - content captured during enumeration"
+          }
+        elsif response.respond_to?(:to_h)
+          begin
+            response.to_h
+          rescue StandardError => e
+            {
+              serialization_error: e.message,
+              class: response.class.name,
+              raw_response: response.to_s
+            }
+          end
+        else
+          # Extract content and token usage information
+          response_data = {}
+          # Get content
+          response_data[:content] = response.content if response.respond_to?(:content)
+          # Extract token usage information
+          response_data[:usage] = extract_usage_metadata(response.usage) if response.respond_to?(:usage)
+          # Extract model information
+          response_data[:model] = response.model if response.respond_to?(:model)
+          # Extract role information
+          response_data[:role] = response.role if response.respond_to?(:role)
+          # Extract ID if available
+          response_data[:id] = response.id if response.respond_to?(:id)
+          # Extract stop reason if available
+          response_data[:stop_reason] = response.stop_reason if response.respond_to?(:stop_reason)
+          # Add class info for debugging
+          response_data[:class] = response.class.name
+          response_data.empty? ? { raw_response: response.to_s, class: response.class.name } : response_data
+        end
+      end
+    end
+    def extract_usage_metadata(usage)
+      if usage.respond_to?(:to_h)
+        usage.to_h
+      elsif usage.is_a?(Hash)
+        usage
+      else
+        # Extract individual usage fields
+        usage_data = {}
+        usage_data[:input_tokens] = usage.input_tokens if usage.respond_to?(:input_tokens)
+        usage_data[:output_tokens] = usage.output_tokens if usage.respond_to?(:output_tokens)
+        usage_data[:total_tokens] = usage_data[:input_tokens].to_i + usage_data[:output_tokens].to_i
+        usage_data
+      end
+    end
+    def clean_request_headers(headers)
+      cleaned = headers.dup
+      # Remove sensitive headers
+      cleaned.delete("Authorization")
+      cleaned.delete("authorization")
+      cleaned.delete("x-api-key")
+      cleaned.delete("X-API-Key")
+      cleaned
+    end
+    def clean_response_headers(headers)
+      # Response headers typically don't contain sensitive data
+      # but we can filter if needed
+      headers.dup
+    end
+    def sanitize_headers(headers)
+      return {} if headers.nil?
+      # Normalize various header-like objects into a Hash{String => String}
+      raw = if headers.is_a?(Hash)
+        headers.transform_keys(&:to_s).transform_values { |v| normalize_header_value(v) }
+      elsif headers.respond_to?(:to_hash)
+        begin
+          headers.to_hash.transform_keys(&:to_s).transform_values { |v| normalize_header_value(v) }
+        rescue StandardError
+          # fall through to other enumeration strategies
+          nil
+        end
+      elsif headers.respond_to?(:each_header)
+        h = {}
+        headers.each_header { |k, v| h[k.to_s] = normalize_header_value(v) }
+        h
+      elsif headers.respond_to?(:each)
+        h = {}
+        headers.each { |k, v| h[k.to_s] = normalize_header_value(v) }
+        h
+      else
+        { "raw" => headers.to_s }
+      end
+      raw ||= {} # in case to_hash raised and nothing was built
+      sanitized = raw.dup
+      sanitized_keys = %w[openai-api-key api-key x-api-key x-goog-api-key]
+      sanitized.each do |k, v|
+        next if v.nil?
+        key_down = k.to_s.downcase
+        if key_down == "authorization"
+          sanitized[k] = if v.to_s.match?(/\ABearer\s+/i)
+            v.to_s.gsub(/\ABearer\s+.+/i, "Bearer [REDACTED]")
+          else
+            "[REDACTED]"
+          end
+        elsif sanitized_keys.include?(key_down)
+          sanitized[k] = "[REDACTED]"
+        end
+      end
+      sanitized
+    end
+    # Helper: convert arrays -> joined string, otherwise to_s
+    def normalize_header_value(value)
+      if value.is_a?(Array)
+        value.join(", ")
+      else
+        value.to_s
+      end
+    end
+    def sanitize_url(url)
+      uri = URI.parse(url)
+      return url unless uri.query
+      sensitive = %w[key api_key apikey token access_token secret]
+      params = URI.decode_www_form(uri.query)
+      redacted = false
+      params.map! do |n, v|
+        if sensitive.include?(n.downcase)
+          redacted = true
+          [n, "[REDACTED]"]
+        else
+          [n, v]
+        end
+      end
+      if redacted
+        uri.query = URI.encode_www_form(params)
+        uri.to_s
+      else
+        url
+      end
+    rescue URI::InvalidURIError
+      url
+    end
+    def send_complete_request_log(request_id:, method:, url:, request_headers:, request_body:, response_headers:,
+      response_body:, status_code:, start_time:, end_time:, duration_ms:, is_streaming:)
+      request_data = {
+        raw_request: {
+          id: request_id,
+          timestamp: start_time.iso8601,
+          method: method.to_s.downcase,
+          url: sanitize_url(url),
+          headers: request_headers,
+          request_body: request_body,
+          response_headers: response_headers,
+          response_body: response_body,
+          status_code: status_code,
+          duration_ms: duration_ms,
+          completed_at: end_time.iso8601,
+          is_streaming: is_streaming
+        }
+      }
+      api_service = Coolhand::ApiService.new
+      api_service.send_llm_request_log(request_data)
+      Coolhand.log "📤 Sent complete request/response log for #{request_id} (duration: #{duration_ms}ms)"
+    rescue StandardError => e
+      Coolhand.log "❌ Error sending complete request log: #{e.message}"
+    end
+    def parse_json(string)
+      JSON.parse(string)
+    rescue JSON::ParserError, TypeError
+      string
+    end
+  end
+end

data/lib/coolhand/collector.rb ADDED Viewed

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+module Coolhand
+  # Utility for generating collector identification string
+  module Collector
+    COLLECTION_METHODS = %w[manual auto-monitor].freeze
+    class << self
+      # Gets the collector identification string
+      # Format: "coolhand-ruby-X.Y.Z" or "coolhand-ruby-X.Y.Z-method"
+      # @param method [String, nil] Optional collection method suffix
+      # @return [String] Collector string identifying this SDK version and collection method
+      def get_collector_string(method = nil)
+        base = "coolhand-ruby-#{VERSION}"
+        method && COLLECTION_METHODS.include?(method) ? "#{base}-#{method}" : base
+      end
+    end
+  end
+end

data/lib/coolhand/configuration.rb ADDED Viewed

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+require "yaml"
+require "uri"
+module Coolhand
+  # Handles all configuration settings for the gem.
+  class Configuration
+    DEFAULT_EXCLUDE_API_PATTERNS = YAML.load_file(
+      File.join(__dir__, "default_exclude_api_patterns.yml")
+    ).freeze
+    DEFAULT_INTERCEPT_ADDRESSES = YAML.load_file(
+      File.join(__dir__, "default_intercept_addresses.yml")
+    ).freeze
+    BASE_URL_ERROR_MSG = "base_url must use https:// (or http://localhost / http://127.0.0.1 for local dev)"
+    LOOPBACK_HOSTS = %w[localhost 127.0.0.1 ::1].freeze
+    attr_accessor :api_key, :environment, :silent, :debug_mode, :capture, :exclude_api_patterns, :enabled
+    attr_reader :intercept_addresses, :base_url
+    def initialize
+      # Set defaults
+      @environment = "production"
+      @api_key = nil
+      @silent = false
+      @intercept_addresses = DEFAULT_INTERCEPT_ADDRESSES.dup
+      self.base_url = "https://coolhandlabs.com/api"
+      @debug_mode = false
+      @capture = true
+      @exclude_api_patterns = DEFAULT_EXCLUDE_API_PATTERNS.dup
+      @enabled = true
+    end
+    # Custom setter that preserves defaults when nil/empty array is provided
+    def intercept_addresses=(value)
+      return if value.nil? || (value.is_a?(Array) && value.empty?)
+      @intercept_addresses = value.is_a?(Array) ? value : [value]
+    end
+    def base_url=(value)
+      stripped = value&.sub(%r{/+\z}, "")
+      raise Error, BASE_URL_ERROR_MSG unless stripped.nil? || valid_base_url?(stripped)
+      @base_url = stripped
+    end
+    def validate!
+      # Validate intercept_addresses after configuration
+      if intercept_addresses.nil? || intercept_addresses.empty?
+        Coolhand.log "❌ Coolhand Error: Intercept addresses cannot be empty. Please set it in the configuration."
+        raise Error, "Intercept addresses cannot be empty"
+      end
+      unless valid_base_url?(base_url)
+        Coolhand.log "❌ Coolhand Error: #{BASE_URL_ERROR_MSG}"
+        raise Error, BASE_URL_ERROR_MSG
+      end
+    end
+    private
+    def valid_base_url?(url)
+      return false if url.nil? || url.empty?
+      parsed = URI.parse(url)
+      host = parsed.hostname&.downcase
+      return false if host.nil? || host.empty?
+      return true if parsed.scheme == "https"
+      parsed.scheme == "http" && LOOPBACK_HOSTS.include?(host)
+    rescue URI::InvalidURIError
+      false
+    end
+  end
+end

data/lib/coolhand/default_exclude_api_patterns.yml ADDED Viewed

@@ -0,0 +1,9 @@
+# Coolhand default exclude API patterns
+# These substrings are matched against request URLs after the intercept_addresses
+# allow-list passes. Matching URLs are skipped and not forwarded as llm_request_logs.
+#
+# Users can extend defaults:   c.exclude_api_patterns << "/myOperationalPath/"
+# Users can override entirely: c.exclude_api_patterns = ["/only_this/"]
+# Users can disable:           c.exclude_api_patterns = []
+- "/batchPredictionJobs/"

data/lib/coolhand/default_intercept_addresses.yml ADDED Viewed

@@ -0,0 +1,19 @@
+# Coolhand default intercept addresses
+# These substrings are matched against request URLs to decide whether a request
+# should be captured and forwarded as an llm_request_log.
+#
+# Users can extend defaults:   c.intercept_addresses << "my.custom.api.com"
+# Users can override entirely: c.intercept_addresses = ["only.this.com"]
+- "api.openai.com"
+- "api.anthropic.com"
+- "api.elevenlabs.io"
+- "generativelanguage.googleapis.com"
+- "models.github.ai"
+- "models.inference.ai.azure.com"
+- ":generateContent"
+- ":streamGenerateContent"
+- "aiplatform.googleapis.com"
+- "gateway.ai.cloudflare.com"
+- "bedrock-runtime"
+- "openrouter.ai"

data/lib/coolhand/feedback_service.rb ADDED Viewed

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+require_relative "api_service"
+module Coolhand
+  class FeedbackService < ApiService
+    def initialize
+      super("v2/llm_request_log_feedbacks")
+    end
+    def create_feedback(feedback)
+      super(feedback, "manual")
+    end
+  end
+end