errsight 0.2.2

data/lib/errsight/client.rb

@@ -0,0 +1,241 @@
+require "net/http"
+require "uri"
+require "json"
+require "openssl"
+
+module Errsight
+  class Client
+    def initialize(config)
+      @config              = config
+      @queue               = []
+      @mutex               = Mutex.new
+      @pid                 = Process.pid
+      @shutdown            = false
+      # Mutex + CV used to interrupt the flush worker's between-tick wait
+      # on shutdown. Plain Thread#wakeup has a lost-wakeup race: if
+      # shutdown! fires before the thread has entered its first sleep,
+      # wakeup is dropped and the thread sleeps the full flush_interval.
+      # CV#wait + setting the flag inside the same lock has no such race.
+      @sleep_mutex         = Mutex.new
+      @sleep_cv            = ConditionVariable.new
+      # When the API returns 429, we set this to a Time and skip sends in
+      # flush! until it elapses. Replaces a per-call `sleep retry_after`
+      # that used to park the flush thread for up to 60s, during which new
+      # events spilled past max_queue_size and got silently dropped.
+      @rate_limited_until  = nil
+      start_flush_worker
+    end
+
+    def enqueue(event)
+      # Fork-safety gate: see detect_fork! comment. Cheapest possible check
+      # since enqueue is on the hot path of every captured event.
+      detect_fork! if @pid != Process.pid
+
+      @mutex.synchronize do
+        if @queue.size >= @config.max_queue_size
+          @config.logger&.warn("[Errsight] Queue full (#{@config.max_queue_size}), dropping event")
+          return
+        end
+        @queue << event
+      end
+    end
+
+    def flush!
+      return if rate_limited?
+      events = nil
+      @mutex.synchronize do
+        return if @queue.empty?
+        events = @queue.slice!(0, @config.batch_size)
+      end
+      send_events(events) if events&.any?
+    end
+
+    def shutdown!
+      # Set the flag and signal under the same lock the flush worker uses
+      # for its CV#wait. This atomically transfers the shutdown signal:
+      # either the worker hasn't entered wait yet (and will see @shutdown
+      # immediately) or it's in wait (and the broadcast unblocks it).
+      @sleep_mutex.synchronize do
+        @shutdown = true
+        @sleep_cv.broadcast
+      end
+      thread = @flush_thread
+      # Bound total shutdown time. If a flush is stuck on a hung HTTP
+      # request, kill it rather than block the host's signal handling.
+      thread&.join(@config.shutdown_timeout)
+      thread&.kill if thread&.alive?
+      # The thread does a final drain on graceful exit; do one here too in
+      # case we had to kill it, or in case shutdown! was called when no
+      # flush thread was alive.
+      flush! rescue nil
+      # Lazy init guard: @http_mutex may be nil if no request ever fired,
+      # in which case the old code raised NoMethodError on shutdown.
+      http_mutex.synchronize { @http&.finish if @http&.started? }
+    rescue StandardError
+      # best-effort
+    end
+
+    private
+
+    # Puma cluster mode (and Spring, Unicorn, custom forking servers) all
+    # boot the Rails app once in the parent and fork worker processes. Only
+    # the calling thread of fork() survives — every other thread, including
+    # our flush worker, vanishes in the child. The Client object is inherited
+    # with a stale @flush_thread reference and a @queue that mirrors the
+    # parent's state at fork time.
+    #
+    # Without this, every event captured in a Puma worker is enqueued into
+    # @queue but nothing ever flushes it. Customers see "the SDK is silently
+    # dropping all my events under cluster mode" with no error in logs.
+    #
+    # Inherited mutexes are also a hazard: if the parent happened to fork
+    # while another thread held @mutex or @http_mutex, the child inherits
+    # them in a stuck-locked state that no thread will ever release. We
+    # rebuild them here so the child starts with clean primitives.
+    #
+    # The inherited @http connection is a duplicated TCP socket fd shared
+    # with the parent; both processes writing to it would interleave bytes
+    # on the wire. Drop it so http_connection lazily reconnects on first
+    # send.
+    #
+    # Race tolerance: two threads in the child can race into detect_fork!
+    # before either has updated @pid, and each spawn a flush thread. Both
+    # survive (same pid_at_start, same @pid) and end up taking turns on
+    # @mutex.synchronize when slicing the queue. No correctness issue —
+    # each batch is handed to exactly one sender via the slice — just a
+    # transient 2x flush rate until one thread is GC'd. The pid_at_start
+    # guard exists for the actual fork case (parent's flush thread doesn't
+    # survive POSIX fork, but defensive against ports/runtimes where it
+    # might).
+    def detect_fork!
+      @queue              = []
+      @http               = nil
+      @mutex              = Mutex.new
+      @http_mutex         = Mutex.new
+      @flush_thread       = nil
+      @pid                = Process.pid
+      # Inherited @shutdown=true from a parent that was shutting down would
+      # cause the child's freshly-spawned flush thread to exit on its first
+      # loop check. Inherited @rate_limited_until from the parent's earlier
+      # 429 has nothing to do with this process's API state. Inherited
+      # @sleep_mutex / @sleep_cv may be in a stuck-locked state if the
+      # parent forked while another thread held them.
+      @shutdown           = false
+      @rate_limited_until = nil
+      @sleep_mutex        = Mutex.new
+      @sleep_cv           = ConditionVariable.new
+      start_flush_worker
+    end
+
+    def start_flush_worker
+      pid_at_start = @pid
+      @flush_thread = Thread.new do
+        loop do
+          @sleep_mutex.synchronize do
+            # CV#wait releases the mutex during the wait and reacquires
+            # before returning. Re-checking @shutdown inside the lock
+            # closes the lost-wakeup race: shutdown! sets @shutdown under
+            # this same lock before broadcasting, so we either see it now
+            # (skip wait) or get woken by the broadcast (and see it after
+            # wait returns).
+            @sleep_cv.wait(@sleep_mutex, @config.flush_interval) unless @shutdown
+          end
+          # shutdown! signaled — exit before flushing again. shutdown! does
+          # a final flush itself, so events queued just before shutdown
+          # aren't lost.
+          break if @shutdown
+          # Orphan-check: a later detect_fork! has installed a newer thread
+          # and overwritten @pid. Older threads (including any that somehow
+          # survived a fork in violation of POSIX) exit instead of fighting
+          # the new thread for queue access.
+          break if @pid != pid_at_start
+          flush! rescue nil
+        end
+        # Graceful drain on the way out so events queued during the
+        # final flush_interval window aren't dropped on a clean shutdown.
+        flush! rescue nil
+      end
+      @flush_thread.abort_on_exception = false
+    end
+
+    def rate_limited?
+      ru = @rate_limited_until
+      ru && Time.now < ru
+    end
+
+    # Cached once at process boot — set_default_paths reads the system trust
+    # store from disk and is expensive to redo per request.
+    def self.cert_store
+      @cert_store ||= OpenSSL::X509::Store.new.tap(&:set_default_paths)
+    end
+
+    def http_mutex
+      @http_mutex ||= Mutex.new
+    end
+
+    def http_connection
+      uri = URI(@config.events_endpoint)
+      if @http && @http.address == uri.host && @http.port == uri.port && @http.started?
+        return @http
+      end
+
+      @http&.finish rescue nil
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == "https"
+      http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      http.cert_store = self.class.cert_store
+      http.open_timeout = @config.timeout
+      http.read_timeout = @config.timeout
+      http.keep_alive_timeout = 30
+      http.start
+      @http = http
+    end
+
+    MAX_PAYLOAD_BYTES = 490 * 1024
+
+    def send_events(events)
+      uri  = URI(@config.events_endpoint)
+      body = JSON.generate(events)
+
+      if body.bytesize > MAX_PAYLOAD_BYTES && events.size > 1
+        mid = (events.size / 2.0).ceil
+        send_events(events.first(mid))
+        send_events(events.last(events.size - mid))
+        return
+      end
+
+      request = Net::HTTP::Post.new(uri.path)
+      request["Content-Type"] = "application/json"
+      request["X-API-Key"] = @config.api_key
+      request["Connection"] = "keep-alive"
+      request.body = body
+
+      response = http_mutex.synchronize { http_connection.request(request) }
+
+      if response.code == "429"
+        retry_after = (response["Retry-After"] || "60").to_i
+        # Cap retry_after so a buggy or malicious upstream returning a
+        # ridiculous Retry-After can't park sends for hours.
+        retry_after = 600 if retry_after > 600
+        @config.logger&.warn("[Errsight] Rate limited — re-queuing #{events.size} events, pausing sends for #{retry_after}s")
+        @mutex.synchronize { @queue.unshift(*events) }
+        # Set a "no-send-until" timestamp instead of sleeping. flush! checks
+        # rate_limited? at the top and skips the slice while we're paused,
+        # so the flush thread keeps ticking and stays responsive to
+        # shutdown! signals.
+        @rate_limited_until = Time.now + retry_after
+        return
+      end
+
+      unless response.is_a?(Net::HTTPSuccess) || response.code == "202"
+        @config.logger&.warn("[Errsight] API error #{response.code}: #{response.body&.slice(0, 200)}")
+      end
+    rescue StandardError => e
+      @config.logger&.warn("[Errsight] Failed to send events: #{e.class}: #{e.message}")
+      http_mutex.synchronize do
+        @http&.finish rescue nil
+        @http = nil
+      end
+    end
+  end
+end

data/lib/errsight/configuration.rb

@@ -0,0 +1,57 @@
+module Errsight
+  class Configuration
+    attr_accessor :api_key, :environment, :min_level, :host, :timeout,
+                  :enabled, :batch_size, :flush_interval, :max_queue_size,
+                  :logger, :attach_to_rails_logger, :release, :shutdown_timeout,
+                  :breadcrumbs_active_record, :breadcrumbs_active_record_capture_binds,
+                  :before_send
+
+    def initialize
+      @api_key       = ENV["ERRSIGHT_API_KEY"]
+      @environment   = ENV.fetch("ERRSIGHT_ENV", "production")
+      @min_level     = :warning
+      @host          = ENV.fetch("ERRSIGHT_HOST", "https://errsight.com")
+      @timeout       = 5
+      @enabled       = true
+      @batch_size    = 10
+      @flush_interval = 2  # seconds
+      @max_queue_size = 1_000
+      @logger                  = defined?(Rails) ? Rails.logger : Logger.new($stdout)
+      # Off by default. When true, every Rails.logger call at min_level or
+      # above becomes an Errsight event, which floods the issue list with
+      # framework deprecation noise and burns customer event quota for
+      # something that should be in their log aggregator, not their error
+      # tracker. Customers who actually want log forwarding can opt in.
+      @attach_to_rails_logger  = false
+      @release                 = ENV["ERRSIGHT_RELEASE"]
+      @shutdown_timeout        = 5  # seconds to wait for flush thread on shutdown
+      # Auto-collect SQL queries as breadcrumbs via the sql.active_record
+      # notification. Default on — this is the headline Rails feature.
+      # Customers with extreme-query-volume workloads can disable.
+      @breadcrumbs_active_record = true
+      # Bind values can carry PII (emails, tokens, customer IDs) so we ship
+      # the parameterized SQL only by default. Compliance-relaxed customers
+      # opt in.
+      @breadcrumbs_active_record_capture_binds = false
+      # Per-event filter. Receives the event hash, must return a (possibly
+      # modified) hash to send, or nil to drop. Used by compliance teams to
+      # scrub PII, by ops teams to drop noisy errors, and by anyone who
+      # wants final-mile control before events leave the process.
+      @before_send                             = nil
+    end
+
+    def enabled?
+      @enabled && !api_key.nil? && !api_key.strip.empty?
+    end
+
+    def validate!
+      if api_key.nil? || api_key.strip.empty?
+        logger&.debug("[Errsight] api_key is not set; Errsight is disabled until one is configured (set ERRSIGHT_API_KEY or config.api_key).")
+      end
+    end
+
+    def events_endpoint
+      "#{host.chomp('/')}/api/v1/events"
+    end
+  end
+end

data/lib/errsight/hub.rb

@@ -0,0 +1,53 @@
+module Errsight
+  # Per-thread scope stack. The top of the stack is the "current" scope read
+  # by Errsight.log when building events.
+  #
+  # Rails request lifecycle pushes a fresh scope in CaptureMiddleware so any
+  # set_user/set_tag/add_breadcrumb calls made during a request are isolated
+  # to that request and don't leak to the next request handled by the same
+  # Puma thread. Same pattern applies to Sidekiq job middleware.
+  class Hub
+    def self.current
+      Thread.current[:errsight_hub] ||= new
+    end
+
+    # Test-only: drop the per-thread hub so each test starts with a clean
+    # scope stack regardless of which thread runs it.
+    def self.reset_current!
+      Thread.current[:errsight_hub] = nil
+    end
+
+    def initialize
+      @stack = [ Scope.new ]
+    end
+
+    def current_scope
+      @stack.last
+    end
+
+    # Push either a fresh fork of the current scope (default) or a specific
+    # scope (used by Sidekiq server middleware to rehydrate scope from a job
+    # payload).
+    def push_scope(scope = nil)
+      @stack.push(scope || current_scope.dup)
+      current_scope
+    end
+
+    def pop_scope
+      # Always keep at least one scope on the stack so callers can never end
+      # up with a nil current_scope after an over-eager pop.
+      @stack.pop if @stack.size > 1
+    end
+
+    # When called with no argument, forks a fresh dup of the current scope
+    # for the block (Rails request middleware, ad-hoc isolation). When called
+    # with an explicit scope, pushes that one as-is — used by Sidekiq server
+    # middleware to install a job's rehydrated scope.
+    def with_scope(scope = nil)
+      push_scope(scope)
+      yield current_scope
+    ensure
+      pop_scope
+    end
+  end
+end

data/lib/errsight/integrations/active_job.rb

@@ -0,0 +1,175 @@
+module Errsight
+  module Integrations
+    # ActiveJob integration. Closes the scope-propagation gap for queue
+    # adapters that aren't Sidekiq — Solid Queue (Rails 8 default),
+    # GoodJob, Delayed::Job's AJ adapter, etc.
+    #
+    # Two responsibilities:
+    #
+    #   1. Snapshot the current scope into the serialized job hash on
+    #      enqueue, rehydrate it on dequeue. Same "user breadcrumbs only"
+    #      rule as Sidekiq client/server middleware (Scope#to_h enforces
+    #      this).
+    #
+    #   2. Capture exceptions raised inside `perform` with structured
+    #      ActiveJob context (job class, queue, executions, adapter).
+    #      Deduped against Rails.error.subscribe + Sidekiq middleware via
+    #      the shared thread-local seen-set.
+    #
+    # When the customer uses Sidekiq+ActiveJob, both our Sidekiq middleware
+    # and this layer fire. The dedup ensures one event per error; the
+    # outermost capturer wins on tags. ActiveJob-wrapped Sidekiq jobs
+    # already get correct unwrapped worker name + args from the Sidekiq
+    # middleware, so the redundancy is harmless.
+    module ActiveJob
+      extend ::ActiveSupport::Concern if defined?(::ActiveSupport::Concern)
+
+      MAX_ARG_BYTES = 4_096
+
+      included do
+        around_perform { |job, block| job.send(:__errsight_around_perform, &block) }
+      end
+
+      # Snapshot scope at enqueue time. ActiveJob calls `serialize` to
+      # produce the wire payload that adapters store; our override
+      # appends a scope key. Adapter-agnostic: works for any backend
+      # that round-trips the serialized hash.
+      def serialize
+        super.tap do |hash|
+          snapshot = Errsight.current_scope.to_h
+          hash["errsight_scope"] = snapshot unless snapshot.empty?
+        end
+      end
+
+      # Capture the snapshot back. ActiveJob calls deserialize before
+      # perform; we stash it on the job instance rather than push to the
+      # hub here because perform happens later (around_perform is the
+      # right place to push so the scope unwinds at perform end).
+      def deserialize(job_data)
+        super
+        @__errsight_scope_snapshot = job_data["errsight_scope"]
+      end
+
+      private
+
+      def __errsight_around_perform
+        scope = __errsight_scope_for_job
+        Errsight.with_scope(scope) do
+          begin
+            yield
+          rescue Exception => exception
+            __errsight_capture_active_job_error(exception)
+            raise
+          end
+        end
+      ensure
+        # Match the dedup-set lifecycle from CaptureMiddleware/Sidekiq:
+        # cleared at job boundary so the next job on this thread starts
+        # fresh.
+        Thread.current[:errsight_captured_exceptions] = nil
+      end
+
+      def __errsight_scope_for_job
+        scope = Errsight.hub.current_scope.dup
+        snapshot = @__errsight_scope_snapshot
+        scope.merge!(::Errsight::Scope.from_h(snapshot)) if snapshot.is_a?(Hash)
+        scope
+      end
+
+      def __errsight_capture_active_job_error(exception)
+        seen = Thread.current[:errsight_captured_exceptions] ||= []
+        return if seen.include?(exception.object_id)
+        seen << exception.object_id
+
+        Errsight.capture_exception(
+          exception,
+          tags:     __errsight_active_job_tags,
+          metadata: { active_job: __errsight_active_job_metadata }
+        )
+      rescue StandardError
+        # Capture failures must never suppress the original exception —
+        # ActiveJob's retry/discard machinery still needs to see it.
+      end
+
+      def __errsight_active_job_tags
+        {
+          "active_job.class"     => self.class.name,
+          "active_job.queue"     => queue_name.to_s,
+          "active_job.job_id"    => job_id.to_s,
+          "active_job.adapter"   => __errsight_adapter_name.to_s
+        }.reject { |_, v| v.to_s.strip.empty? }
+      end
+
+      def __errsight_active_job_metadata
+        {
+          "arguments"    => __errsight_filter_arguments(arguments),
+          "executions"   => respond_to?(:executions) ? executions : nil,
+          "enqueued_at"  => respond_to?(:enqueued_at) ? enqueued_at&.iso8601(3) : nil,
+          "scheduled_at" => respond_to?(:scheduled_at) ? scheduled_at&.iso8601(3) : nil,
+          "queue_name"   => queue_name,
+          "priority"     => respond_to?(:priority) ? priority : nil
+        }.compact
+      end
+
+      # Different Rails / ActiveJob versions expose adapter naming
+      # differently. Try the canonical method, fall back to inferring
+      # from the adapter class name. Wrapped in rescue because some
+      # custom adapters can raise on these accessors.
+      def __errsight_adapter_name
+        if self.class.respond_to?(:queue_adapter_name)
+          self.class.queue_adapter_name
+        else
+          self.class.queue_adapter.class.name.to_s.split("::").last.sub(/Adapter\z/, "").downcase
+        end
+      rescue StandardError
+        "unknown"
+      end
+
+      # Same PII strategy as Sidekiq middleware: walk hashes through
+      # ActiveSupport::ParameterFilter so the host's filter_parameters
+      # config (passwords, tokens, ssn fields) is honored. Cap value
+      # sizes defensively — ActiveJob args can include serialized
+      # ActiveRecord objects via GlobalID, which are short, but custom
+      # serializers might emit large blobs.
+      def __errsight_filter_arguments(args)
+        return [] unless args.is_a?(Array)
+        args.map { |a| __errsight_filter_one(a) }
+      end
+
+      def __errsight_filter_one(value)
+        case value
+        when Hash  then __errsight_filter_hash(value)
+        when Array then value.map { |a| __errsight_filter_one(a) }
+        else            __errsight_truncate(value)
+        end
+      end
+
+      def __errsight_filter_hash(hash)
+        filter = self.class.send(:__errsight_parameter_filter)
+        normalized = hash.transform_keys(&:to_s)
+        filtered = filter ? filter.filter(normalized) : normalized
+        filtered.transform_values { |v| __errsight_filter_one(v) }
+      rescue StandardError
+        { "_unfilterable" => "[#{hash.class.name}]" }
+      end
+
+      def __errsight_truncate(value)
+        str = value.to_s
+        return value if str.bytesize <= MAX_ARG_BYTES
+        "[truncated #{value.class.name} #{str.bytesize}b]"
+      end
+
+      class_methods do
+        # Cached at class level — ParameterFilter rebuilds aren't free
+        # and the filter list doesn't change at runtime.
+        def __errsight_parameter_filter
+          return @__errsight_parameter_filter if defined?(@__errsight_parameter_filter)
+          @__errsight_parameter_filter =
+            if defined?(::ActiveSupport::ParameterFilter) && defined?(::Rails) && ::Rails.application
+              ::ActiveSupport::ParameterFilter.new(::Rails.application.config.filter_parameters)
+            end
+        end
+      end
+    end
+  end
+end

data/lib/errsight/integrations/active_record.rb

@@ -0,0 +1,94 @@
+require "active_support/notifications"
+
+module Errsight
+  module Integrations
+    # Subscribes to sql.active_record and pushes each non-cached, non-schema
+    # query into the current scope's DB breadcrumb ring. When an exception
+    # is later captured in the same request/job, the event ships with the
+    # last 30 queries that ran — turning "ActiveRecord::RecordNotFound" from
+    # an opaque stack trace into "we ran these 5 queries before failing on
+    # this one."
+    #
+    # This is the killer demo for the Rails wedge. Sentry-ruby has SQL
+    # breadcrumbs but they come from a generic Notifications layer that
+    # treats every framework the same; here it's first-class and tuned for
+    # what Rails apps actually emit.
+    module ActiveRecord
+      MAX_SQL_BYTES = 2_048
+      MAX_BIND_VALUES = 20
+      INTERNAL_NAMES = %w[SCHEMA TRANSACTION].freeze
+
+      class << self
+        # Idempotent. The subscriber handle is stored at module level so a
+        # double-require (gem reloaded in dev, or required from both the
+        # Railtie and a manual `require`) doesn't double-fire crumbs.
+        def subscribe!
+          return @subscriber if @subscriber
+          @subscriber = ::ActiveSupport::Notifications.subscribe("sql.active_record") do |*args|
+            handle(::ActiveSupport::Notifications::Event.new(*args))
+          end
+        end
+
+        def unsubscribe!
+          if @subscriber
+            ::ActiveSupport::Notifications.unsubscribe(@subscriber)
+            @subscriber = nil
+          end
+        end
+
+        private
+
+        def handle(event)
+          payload = event.payload
+          return if INTERNAL_NAMES.include?(payload[:name])
+          return if payload[:cached]
+
+          sql = payload[:sql].to_s
+          return if sql.empty?
+          if sql.bytesize > MAX_SQL_BYTES
+            sql = sql.byteslice(0, MAX_SQL_BYTES) + "…[truncated]"
+          end
+
+          duration_ms = event.duration.round(2)
+          name        = payload[:name].to_s
+          message     = name.empty? ? "SQL (#{duration_ms}ms)" : "#{name} (#{duration_ms}ms)"
+
+          data = { sql: sql, duration_ms: duration_ms }
+          data[:name]          = name                       unless name.empty?
+          data[:connection_id] = payload[:connection_id]    if payload[:connection_id]
+
+          if Errsight.configuration.breadcrumbs_active_record_capture_binds
+            binds = extract_binds(payload[:type_casted_binds] || payload[:binds])
+            data[:binds] = binds unless binds.empty?
+          end
+
+          Errsight.current_scope.add_db_breadcrumb(message: message, data: data)
+        rescue StandardError
+          # Never let our subscriber take down the host's request or job —
+          # a missing breadcrumb is far less bad than a crashed page.
+        end
+
+        # Bind values across Rails versions:
+        #   - 7.x+: ActiveModel::Attribute instances responding to #value
+        #   - older: [ActiveRecord::ConnectionAdapters::Column, value] tuples
+        #   - type_casted_binds: already plain values (preferred when set)
+        # Cap to MAX_BIND_VALUES so a SELECT IN (1..10000) doesn't ship 10k
+        # values per breadcrumb.
+        def extract_binds(binds)
+          return [] unless binds.is_a?(Array)
+          binds.first(MAX_BIND_VALUES).map do |b|
+            if b.respond_to?(:value)
+              b.value
+            elsif b.is_a?(Array)
+              b.last
+            else
+              b
+            end
+          end
+        rescue StandardError
+          []
+        end
+      end
+    end
+  end
+end