vectra-client 0.3.0 → 0.3.2

data/lib/vectra/logging.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+require "json"
+require "logger"
+
+module Vectra
+  # Structured JSON logger for Vectra operations
+  #
+  # Provides consistent, machine-readable logging for all Vectra operations.
+  # Output is JSON formatted for easy parsing by log aggregators.
+  #
+  # @example Basic usage
+  #   Vectra.configure do |config|
+  #     config.logger = Vectra::JsonLogger.new(STDOUT)
+  #   end
+  #
+  # @example With file output
+  #   Vectra.configure do |config|
+  #     config.logger = Vectra::JsonLogger.new("log/vectra.log")
+  #   end
+  #
+  # @example With custom metadata
+  #   logger = Vectra::JsonLogger.new(STDOUT, app: "my-app", env: "production")
+  #
+  class JsonLogger
+    SEVERITY_LABELS = {
+      Logger::DEBUG => "debug",
+      Logger::INFO => "info",
+      Logger::WARN => "warn",
+      Logger::ERROR => "error",
+      Logger::FATAL => "fatal"
+    }.freeze
+
+    attr_reader :output, :default_metadata
+
+    # Initialize JSON logger
+    #
+    # @param output [IO, String] Output destination (IO object or file path)
+    # @param metadata [Hash] Default metadata to include in all log entries
+    def initialize(output = $stdout, **metadata)
+      @output = resolve_output(output)
+      @default_metadata = metadata
+      @mutex = Mutex.new
+    end
+
+    # Log debug message
+    #
+    # @param message [String] Log message
+    # @param data [Hash] Additional data
+    def debug(message, **data)
+      log(Logger::DEBUG, message, **data)
+    end
+
+    # Log info message
+    #
+    # @param message [String] Log message
+    # @param data [Hash] Additional data
+    def info(message, **data)
+      log(Logger::INFO, message, **data)
+    end
+
+    # Log warning message
+    #
+    # @param message [String] Log message
+    # @param data [Hash] Additional data
+    def warn(message, **data)
+      log(Logger::WARN, message, **data)
+    end
+
+    # Log error message
+    #
+    # @param message [String] Log message
+    # @param data [Hash] Additional data
+    def error(message, **data)
+      log(Logger::ERROR, message, **data)
+    end
+
+    # Log fatal message
+    #
+    # @param message [String] Log message
+    # @param data [Hash] Additional data
+    def fatal(message, **data)
+      log(Logger::FATAL, message, **data)
+    end
+
+    # Log Vectra operation event
+    #
+    # @param event [Instrumentation::Event] Operation event
+    def log_operation(event)
+      data = {
+        provider: event.provider.to_s,
+        operation: event.operation.to_s,
+        index: event.index,
+        duration_ms: event.duration,
+        success: event.success?
+      }
+
+      # Add metadata
+      data[:vector_count] = event.metadata[:vector_count] if event.metadata[:vector_count]
+      data[:result_count] = event.metadata[:result_count] if event.metadata[:result_count]
+
+      # Add error info if present
+      if event.error
+        data[:error_class] = event.error.class.name
+        data[:error_message] = event.error.message
+      end
+
+      level = event.success? ? Logger::INFO : Logger::ERROR
+      log(level, "vectra.#{event.operation}", **data)
+    end
+
+    # Close the logger
+    #
+    # @return [void]
+    def close
+      @output.close if @output.respond_to?(:close) && @output != $stdout && @output != $stderr
+    end
+
+    private
+
+    def log(severity, message, **data)
+      entry = build_entry(severity, message, data)
+
+      @mutex.synchronize do
+        @output.puts(JSON.generate(entry))
+        @output.flush if @output.respond_to?(:flush)
+      end
+    end
+
+    def build_entry(severity, message, data)
+      {
+        timestamp: Time.now.utc.iso8601(3),
+        level: SEVERITY_LABELS[severity],
+        logger: "vectra",
+        message: message
+      }.merge(default_metadata).merge(data).compact
+    end
+
+    def resolve_output(output)
+      case output
+      when IO, StringIO
+        output
+      when String
+        File.open(output, "a")
+      else
+        $stdout
+      end
+    end
+  end
+
+  # Instrumentation handler for JSON logging
+  #
+  # @example Enable JSON logging
+  #   require 'vectra/logging'
+  #
+  #   Vectra::Logging.setup!(
+  #     output: "log/vectra.json.log",
+  #     app: "my-service"
+  #   )
+  #
+  module Logging
+    class << self
+      attr_reader :logger
+
+      # Setup JSON logging for Vectra
+      #
+      # @param output [IO, String] Log output
+      # @param metadata [Hash] Default metadata
+      # @return [JsonLogger]
+      def setup!(output: $stdout, **metadata)
+        @logger = JsonLogger.new(output, **metadata)
+
+        # Register as instrumentation handler
+        Vectra::Instrumentation.on_operation do |event|
+          @logger.log_operation(event)
+        end
+
+        # Also set as Vectra's logger for retry/error logging
+        Vectra.configuration.logger = @logger
+
+        @logger
+      end
+
+      # Log a custom event
+      #
+      # @param level [Symbol] Log level (:debug, :info, :warn, :error, :fatal)
+      # @param message [String] Log message
+      # @param data [Hash] Additional data
+      def log(level, message, **data)
+        return unless @logger
+
+        @logger.public_send(level, message, **data)
+      end
+    end
+  end
+
+  # Log formatter for standard Ruby Logger that outputs JSON
+  #
+  # @example With standard Logger
+  #   logger = Logger.new(STDOUT)
+  #   logger.formatter = Vectra::JsonFormatter.new(app: "my-app")
+  #
+  class JsonFormatter
+    attr_reader :default_metadata
+
+    def initialize(**metadata)
+      @default_metadata = metadata
+    end
+
+    def call(severity, time, _progname, message)
+      entry = {
+        timestamp: time.utc.iso8601(3),
+        level: severity.downcase,
+        logger: "vectra",
+        message: format_message(message)
+      }.merge(default_metadata)
+
+      # Parse structured data if message is a hash
+      if message.is_a?(Hash)
+        entry.merge!(message)
+        entry[:message] = message[:message] || message[:msg] || "operation"
+      end
+
+      "#{JSON.generate(entry.compact)}\n"
+    end
+
+    private
+
+    def format_message(message)
+      case message
+      when String
+        message
+      when Exception
+        "#{message.class}: #{message.message}"
+      when Hash
+        message[:message] || message.to_s
+      else
+        message.to_s
+      end
+    end
+  end
+end

data/lib/vectra/rate_limiter.rb

@@ -0,0 +1,304 @@
+# frozen_string_literal: true
+
+module Vectra
+  # Proactive rate limiter using token bucket algorithm
+  #
+  # Throttles requests BEFORE sending to prevent rate limit errors from providers.
+  # Uses token bucket algorithm for smooth rate limiting with burst support.
+  #
+  # @example Basic usage
+  #   limiter = Vectra::RateLimiter.new(
+  #     requests_per_second: 10,
+  #     burst_size: 20
+  #   )
+  #
+  #   limiter.acquire do
+  #     client.query(...)
+  #   end
+  #
+  # @example With client wrapper
+  #   client = Vectra::RateLimitedClient.new(
+  #     Vectra::Client.new(...),
+  #     requests_per_second: 10
+  #   )
+  #   client.query(...)  # Automatically rate limited
+  #
+  class RateLimiter
+    class RateLimitExceededError < Vectra::Error
+      attr_reader :wait_time
+
+      def initialize(wait_time:)
+        @wait_time = wait_time
+        super("Rate limit exceeded. Retry after #{wait_time.round(2)} seconds")
+      end
+    end
+
+    attr_reader :requests_per_second, :burst_size
+
+    # Initialize rate limiter
+    #
+    # @param requests_per_second [Float] Sustained request rate
+    # @param burst_size [Integer] Maximum burst capacity (default: 2x RPS)
+    def initialize(requests_per_second:, burst_size: nil)
+      @requests_per_second = requests_per_second.to_f
+      @burst_size = burst_size || (requests_per_second * 2).to_i
+      @tokens = @burst_size.to_f
+      @last_refill = Time.now
+      @mutex = Mutex.new
+    end
+
+    # Acquire a token and execute block
+    #
+    # @param wait [Boolean] Wait for token if not available (default: true)
+    # @param timeout [Float] Maximum wait time in seconds (default: 30)
+    # @yield Block to execute after acquiring token
+    # @return [Object] Result of block
+    # @raise [RateLimitExceededError] If wait is false and no token available
+    def acquire(wait: true, timeout: 30, &)
+      acquired = try_acquire(wait: wait, timeout: timeout)
+
+      unless acquired
+        wait_time = time_until_token
+        raise RateLimitExceededError.new(wait_time: wait_time)
+      end
+
+      yield
+    end
+
+    # Try to acquire a token without blocking
+    #
+    # @return [Boolean] true if token acquired
+    def try_acquire(wait: false, timeout: 30)
+      deadline = Time.now + timeout
+
+      loop do
+        @mutex.synchronize do
+          refill_tokens
+          if @tokens >= 1
+            @tokens -= 1
+            return true
+          end
+        end
+
+        return false unless wait
+        return false if Time.now >= deadline
+
+        # Wait a bit before retrying
+        sleep([time_until_token, 0.1].min)
+      end
+    end
+
+    # Get current token count
+    #
+    # @return [Float]
+    def available_tokens
+      @mutex.synchronize do
+        refill_tokens
+        @tokens
+      end
+    end
+
+    # Get time until next token is available
+    #
+    # @return [Float] Seconds until next token
+    def time_until_token
+      @mutex.synchronize do
+        refill_tokens
+        return 0 if @tokens >= 1
+
+        tokens_needed = 1 - @tokens
+        tokens_needed / @requests_per_second
+      end
+    end
+
+    # Get rate limiter statistics
+    #
+    # @return [Hash]
+    def stats
+      @mutex.synchronize do
+        refill_tokens
+        {
+          requests_per_second: @requests_per_second,
+          burst_size: @burst_size,
+          available_tokens: @tokens.round(2),
+          time_until_token: @tokens >= 1 ? 0 : ((1 - @tokens) / @requests_per_second).round(3)
+        }
+      end
+    end
+
+    # Reset the rate limiter to full capacity
+    #
+    # @return [void]
+    def reset!
+      @mutex.synchronize do
+        @tokens = @burst_size.to_f
+        @last_refill = Time.now
+      end
+    end
+
+    private
+
+    def refill_tokens
+      now = Time.now
+      elapsed = now - @last_refill
+      @last_refill = now
+
+      # Add tokens based on elapsed time
+      new_tokens = elapsed * @requests_per_second
+      @tokens = [@tokens + new_tokens, @burst_size.to_f].min
+    end
+  end
+
+  # Client wrapper with automatic rate limiting
+  #
+  # @example
+  #   client = Vectra::RateLimitedClient.new(
+  #     Vectra::Client.new(provider: :pinecone, ...),
+  #     requests_per_second: 10,
+  #     burst_size: 20
+  #   )
+  #
+  #   client.query(...)  # Automatically waits if rate limited
+  #
+  class RateLimitedClient
+    RATE_LIMITED_METHODS = [:upsert, :query, :fetch, :update, :delete].freeze
+
+    attr_reader :client, :limiter
+
+    # Initialize rate limited client
+    #
+    # @param client [Client] The underlying Vectra client
+    # @param requests_per_second [Float] Request rate limit
+    # @param burst_size [Integer] Burst capacity
+    # @param wait [Boolean] Wait for rate limit (default: true)
+    def initialize(client, requests_per_second:, burst_size: nil, wait: true)
+      @client = client
+      @limiter = RateLimiter.new(
+        requests_per_second: requests_per_second,
+        burst_size: burst_size
+      )
+      @wait = wait
+    end
+
+    # Rate-limited upsert
+    def upsert(...)
+      with_rate_limit { client.upsert(...) }
+    end
+
+    # Rate-limited query
+    def query(...)
+      with_rate_limit { client.query(...) }
+    end
+
+    # Rate-limited fetch
+    def fetch(...)
+      with_rate_limit { client.fetch(...) }
+    end
+
+    # Rate-limited update
+    def update(...)
+      with_rate_limit { client.update(...) }
+    end
+
+    # Rate-limited delete
+    def delete(...)
+      with_rate_limit { client.delete(...) }
+    end
+
+    # Pass through other methods without rate limiting
+    def method_missing(method, *, **, &)
+      if client.respond_to?(method)
+        client.public_send(method, *, **, &)
+      else
+        super
+      end
+    end
+
+    def respond_to_missing?(method, include_private = false)
+      client.respond_to?(method, include_private) || super
+    end
+
+    # Get rate limiter stats
+    #
+    # @return [Hash]
+    def rate_limit_stats
+      limiter.stats
+    end
+
+    private
+
+    def with_rate_limit(&)
+      limiter.acquire(wait: @wait, &)
+    end
+  end
+
+  # Per-provider rate limiter registry
+  #
+  # @example
+  #   # Configure rate limits per provider
+  #   Vectra::RateLimiterRegistry.configure(:pinecone, requests_per_second: 100)
+  #   Vectra::RateLimiterRegistry.configure(:qdrant, requests_per_second: 50)
+  #
+  #   # Get limiter for provider
+  #   limiter = Vectra::RateLimiterRegistry[:pinecone]
+  #   limiter.acquire { ... }
+  #
+  module RateLimiterRegistry
+    class << self
+      # Configure rate limiter for provider
+      #
+      # @param provider [Symbol] Provider name
+      # @param requests_per_second [Float] Request rate
+      # @param burst_size [Integer] Burst capacity
+      # @return [RateLimiter]
+      def configure(provider, requests_per_second:, burst_size: nil)
+        limiters[provider.to_sym] = RateLimiter.new(
+          requests_per_second: requests_per_second,
+          burst_size: burst_size
+        )
+      end
+
+      # Get limiter for provider
+      #
+      # @param provider [Symbol] Provider name
+      # @return [RateLimiter, nil]
+      def [](provider)
+        limiters[provider.to_sym]
+      end
+
+      # Get all configured limiters
+      #
+      # @return [Hash<Symbol, RateLimiter>]
+      def all
+        limiters.dup
+      end
+
+      # Get stats for all limiters
+      #
+      # @return [Hash<Symbol, Hash>]
+      def stats
+        limiters.transform_values(&:stats)
+      end
+
+      # Reset all limiters
+      #
+      # @return [void]
+      def reset_all!
+        limiters.each_value(&:reset!)
+      end
+
+      # Clear all limiters
+      #
+      # @return [void]
+      def clear!
+        @limiters = {}
+      end
+
+      private
+
+      def limiters
+        @limiters ||= {}
+      end
+    end
+  end
+end

data/lib/vectra/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Vectra
-  VERSION = "0.3.0"
+  VERSION = "0.3.2"
 end

data/lib/vectra.rb

@@ -11,6 +11,12 @@ require_relative "vectra/batch"
 require_relative "vectra/streaming"
 require_relative "vectra/cache"
 require_relative "vectra/pool"
+require_relative "vectra/circuit_breaker"
+require_relative "vectra/rate_limiter"
+require_relative "vectra/logging"
+require_relative "vectra/health_check"
+require_relative "vectra/credential_rotation"
+require_relative "vectra/audit_log"
 require_relative "vectra/active_record"
 require_relative "vectra/providers/base"
 require_relative "vectra/providers/pinecone"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: vectra-client
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.2
 platform: ruby
 authors:
 - Mijo Kristo
@@ -258,7 +258,13 @@ files:
 - docs/examples/index.md
 - docs/guides/getting-started.md
 - docs/guides/installation.md
+- docs/guides/monitoring.md
 - docs/guides/performance.md
+- docs/guides/runbooks/cache-issues.md
+- docs/guides/runbooks/high-error-rate.md
+- docs/guides/runbooks/high-latency.md
+- docs/guides/runbooks/pool-exhausted.md
+- docs/guides/security.md
 - docs/index.md
 - docs/providers/index.md
 - docs/providers/pgvector.md
@@ -272,14 +278,21 @@ files:
 - lib/generators/vectra/templates/vectra.rb
 - lib/vectra.rb
 - lib/vectra/active_record.rb
+- lib/vectra/audit_log.rb
 - lib/vectra/batch.rb
 - lib/vectra/cache.rb
+- lib/vectra/circuit_breaker.rb
 - lib/vectra/client.rb
 - lib/vectra/configuration.rb
+- lib/vectra/credential_rotation.rb
 - lib/vectra/errors.rb
+- lib/vectra/health_check.rb
 - lib/vectra/instrumentation.rb
 - lib/vectra/instrumentation/datadog.rb
+- lib/vectra/instrumentation/honeybadger.rb
 - lib/vectra/instrumentation/new_relic.rb
+- lib/vectra/instrumentation/sentry.rb
+- lib/vectra/logging.rb
 - lib/vectra/pool.rb
 - lib/vectra/providers/base.rb
 - lib/vectra/providers/pgvector.rb
@@ -290,6 +303,7 @@ files:
 - lib/vectra/providers/qdrant.rb
 - lib/vectra/providers/weaviate.rb
 - lib/vectra/query_result.rb
+- lib/vectra/rate_limiter.rb
 - lib/vectra/retry.rb
 - lib/vectra/streaming.rb
 - lib/vectra/vector.rb