errorgap 0.7.0 → 0.9.0

data/lib/errorgap/log_delivery.rb

@@ -1,8 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-require "net/http"
-require "uri"
 require "time"
 
 module Errorgap
@@ -39,27 +36,13 @@ module Errorgap
       if sync || !@configuration.async
         deliver(payload)
       else
-        Errorgap.register_thread(Thread.new { deliver(payload) })
+        Errorgap.delivery.enqueue("logs", payload)
         nil
       end
     end
 
     def deliver(payload)
-      uri = URI.join(
-        @configuration.endpoint.end_with?("/") ? @configuration.endpoint : "#{@configuration.endpoint}/",
-        "api/projects/#{@configuration.project_slug}/logs"
-      )
-      request = Net::HTTP::Post.new(uri)
-      request["Content-Type"] = "application/json"
-      request["User-Agent"] = "errorgap-ruby/#{Errorgap::VERSION}"
-      request["X-Errorgap-Project-Key"] = @configuration.api_key if present?(@configuration.api_key)
-      request.body = JSON.generate(payload)
-
-      Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
-        http.request(request)
-      end
-    rescue StandardError => exception
-      @configuration.logger&.warn("[errorgap] log delivery error: #{exception.class}: #{exception.message}")
+      Errorgap.delivery.post("logs", payload)
     end
 
     def self.normalize_level(level)
@@ -71,11 +54,5 @@ module Errorgap
     def self.rank(level)
       LEVELS.index(level) || LEVELS.index("info")
     end
-
-    private
-
-    def present?(value)
-      !value.nil? && !value.to_s.empty?
-    end
   end
 end

data/lib/errorgap/notice.rb

@@ -11,6 +11,7 @@ module Errorgap
     MAX_SOURCE_FRAMES = 25
     MAX_SOURCE_LINE_LENGTH = 400
     MAX_CAUSE_DEPTH = 10
+    MAX_MESSAGE_LENGTH = 8192
 
     def self.from_exception(error, configuration:, context: {}, environment: {}, session: {}, params: {}, breadcrumbs: [])
       new(
@@ -40,15 +41,15 @@ module Errorgap
         received_at: Time.now.utc.iso8601,
         errors: [
           {
-            type: @error.class.name,
-            message: @error.message.to_s,
+            type: @error.class.name.to_s,
+            message: Sanitizer.string(@error.message, MAX_MESSAGE_LENGTH),
             backtrace: backtrace_frames
           }
         ],
-        context: default_context.merge(stringify_hash(@context)),
-        environment: stringify_hash(@environment),
-        session: stringify_hash(@session),
-        params: filter_hash(@params)
+        context: default_context.merge(sanitizer.clean(@context)),
+        environment: sanitizer.clean(@environment),
+        session: sanitizer.filter(@session),
+        params: sanitizer.filter(@params)
       }
     end
 
@@ -100,7 +101,7 @@ module Errorgap
     # The causes beyond the root error, as {type, message} pairs.
     def collect_causes
       error_chain.drop(1).map do |link|
-        { type: link.class.name, message: link.message.to_s }
+        { type: link.class.name.to_s, message: Sanitizer.string(link.message, MAX_MESSAGE_LENGTH) }
       end
     end
 
@@ -173,20 +174,8 @@ module Errorgap
       absolute_path.sub(%r{\A#{Regexp.escape(root)}/?}, "")
     end
 
-    def filter_hash(hash)
-      stringify_hash(hash).each_with_object({}) do |(key, value), filtered|
-        filtered[key] = sensitive?(key) ? "[FILTERED]" : value
-      end
-    end
-
-    def sensitive?(key)
-      @configuration.filter_keys.any? { |filter| key.to_s.downcase.include?(filter.downcase) }
-    end
-
-    def stringify_hash(hash)
-      hash.each_with_object({}) do |(key, value), result|
-        result[key.to_s] = value.is_a?(Hash) ? stringify_hash(value) : value
-      end
+    def sanitizer
+      @sanitizer ||= Sanitizer.new(@configuration.filter_keys)
     end
   end
 end

data/lib/errorgap/notifier.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-require "net/http"
-require "uri"
-
 module Errorgap
   class Notifier
     Response = Struct.new(:status, :body, :error, keyword_init: true) do
@@ -33,11 +29,15 @@ module Errorgap
         params: params,
         breadcrumbs: breadcrumbs
       )
+      payload = run_hooks(notice.to_h)
+      return Response.new(status: 202, body: "dropped by before_notify") unless payload
+
+      Errorgap.record_release_once if @configuration.track_release
 
       if sync || !@configuration.async
-        deliver(notice)
+        deliver(payload)
       else
-        Errorgap.register_thread(Thread.new { deliver(notice) })
+        Errorgap.delivery.enqueue("notices", payload)
         Response.new(status: 202, body: "queued")
       end
     rescue StandardError => exception
@@ -45,18 +45,11 @@ module Errorgap
       Response.new(error: exception)
     end
 
+    # `notice` is a Notice or an already built payload Hash.
     def deliver(notice)
-      uri = URI.join(@configuration.endpoint.end_with?("/") ? @configuration.endpoint : "#{@configuration.endpoint}/", "api/projects/#{@configuration.project_slug}/notices")
-      request = Net::HTTP::Post.new(uri)
-      request["Content-Type"] = "application/json"
-      request["User-Agent"] = "errorgap-ruby/#{Errorgap::VERSION}"
-      request["X-Errorgap-Project-Key"] = @configuration.api_key if present?(@configuration.api_key)
-      request.body = JSON.generate(notice.to_h)
-
-      response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
-        http.request(request)
-      end
-      Response.new(status: response.code.to_i, body: response.body)
+      payload = notice.respond_to?(:to_h) && !notice.is_a?(Hash) ? notice.to_h : notice
+      result = Errorgap.delivery.post("notices", payload)
+      Response.new(status: result.status, body: result.body, error: result.error)
     rescue StandardError => exception
       log(exception)
       Response.new(error: exception)
@@ -64,12 +57,25 @@ module Errorgap
 
     private
 
-    def log(exception)
-      @configuration.logger&.warn("[errorgap] #{exception.class}: #{exception.message}")
+    # A hook that raises is logged and skipped: a bug in a filter must not
+    # lose the error it was filtering.
+    def run_hooks(payload)
+      @configuration.before_notify_hooks.each do |hook|
+        begin
+          result = hook.call(payload)
+        rescue StandardError => exception
+          log(exception)
+          next
+        end
+        return nil if result == false
+
+        payload = result if result.is_a?(Hash) && result.key?(:errors)
+      end
+      payload
     end
 
-    def present?(value)
-      !value.nil? && !value.to_s.empty?
+    def log(exception)
+      @configuration.logger&.warn("[errorgap] #{exception.class}: #{exception.message}")
     end
   end
 end

data/lib/errorgap/rack_middleware.rb

@@ -20,6 +20,7 @@ module Errorgap
           notify_once(env, exception)
           raise
         ensure
+          Errorgap.clear_user
           elapsed_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000.0
           record_transaction(env, status || 500, elapsed_ms, transaction_id)
         end
@@ -33,6 +34,8 @@ module Errorgap
     rescue Exception => exception # rubocop:disable Lint/RescueException
       notify_once(env, exception)
       raise
+    ensure
+      Errorgap.clear_user
     end
 
     UUID = /\A\h{8}-\h{4}-\h{4}-\h{4}-\h{12}\z/
@@ -85,18 +88,19 @@ module Errorgap
       env["PATH_INFO"]
     end
 
+    # Queued, not sent inline: a slow or unreachable errorgap must not add
+    # its timeout to a request that is already failing.
     def notify_once(env, exception)
       notified = env["errorgap.notified_exception_ids"] ||= {}
       return if notified[exception.object_id]
 
       notified[exception.object_id] = true
-      Errorgap.notify(
+      Errorgap.notify_once(
         exception,
         context: rack_context(env),
         environment: rack_environment(env),
         session: rack_session(env),
-        params: rack_params(env),
-        sync: true
+        params: rack_params(env)
       )
     end
 
@@ -112,15 +116,54 @@ module Errorgap
       {
         method: env["REQUEST_METHOD"],
         path: env["PATH_INFO"],
-        query_string: env["QUERY_STRING"],
+        query_string: filtered_query(env["QUERY_STRING"]),
         user_agent: env["HTTP_USER_AGENT"],
         remote_addr: env["REMOTE_ADDR"]
       }
     end
 
+    # `?token=abc` is as sensitive in the query string as it is in params.
+    def filtered_query(query)
+      return query if query.nil? || query.empty?
+
+      keys = Array(Errorgap.configuration.filter_keys)
+      query.split("&").map do |pair|
+        name, value = pair.split("=", 2)
+        sensitive = keys.any? do |key|
+          key.is_a?(Regexp) ? key.match?(name.to_s) : name.to_s.downcase.include?(key.to_s.downcase)
+        end
+        sensitive && value ? "#{name}=[FILTERED]" : pair
+      end.join("&")
+    rescue StandardError
+      nil
+    end
+
     def rack_session(env)
       session = env["rack.session"]
-      session.respond_to?(:to_hash) ? session.to_hash : {}
+      data = session.respond_to?(:to_hash) ? session.to_hash : {}
+      signed_in_user(env).merge(data)
+    rescue StandardError
+      {}
+    end
+
+    # The signed-in user (Warden / Devise), unless the app named one itself
+    # with `Errorgap.set_user`. Only looks at a user Warden already loaded:
+    # it never triggers authentication from inside error handling.
+    def signed_in_user(env)
+      return {} unless Errorgap.configuration.capture_user
+      return {} if Errorgap.current_user
+
+      warden = env["warden"]
+      return {} unless warden.respond_to?(:authenticated?) && warden.respond_to?(:user)
+      return {} unless warden.authenticated?
+
+      user = warden.user
+      found = {}
+      found["user_id"] = user.id.to_s if user.respond_to?(:id) && user.id
+      found["user_email"] = user.email.to_s if user.respond_to?(:email) && user.email
+      found
+    rescue StandardError
+      {}
     end
 
     def rack_params(env)

data/lib/errorgap/rails/railtie.rb

@@ -23,6 +23,19 @@ module Errorgap
         # subscribers no-op unless the middleware starts a per-request
         # collection, which does check the flag.
         SpanCollector.install
+        Errorgap::Integrations::NetHTTP.install
+      end
+
+      # Background jobs: report their errors and time them. Like the span
+      # collector these check the configuration when a job runs.
+      initializer "errorgap.install_job_integrations" do
+        Errorgap::Integrations::ActiveJob.install
+        Errorgap::Integrations::Sidekiq.install
+      end
+
+      # After config/initializers, so `auth_events` is readable here.
+      config.after_initialize do
+        Errorgap::WardenHooks.install if Errorgap.configuration.auth_events
       end
 
       generators do

data/lib/errorgap/release_detector.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Errorgap
+  # Works out which commit is running when `config.release` is not set, so
+  # errors are attributed to a deploy without any setup: the commit variables
+  # hosting platforms export, Capistrano's REVISION file, then the checkout's
+  # `.git/HEAD`.
+  module ReleaseDetector
+    ENV_KEYS = %w[
+      SOURCE_VERSION HEROKU_SLUG_COMMIT HEROKU_BUILD_COMMIT RENDER_GIT_COMMIT
+      RAILWAY_GIT_COMMIT_SHA VERCEL_GIT_COMMIT_SHA KAMAL_VERSION GIT_COMMIT GIT_SHA
+    ].freeze
+
+    class << self
+      def detect(root_directory, env = ENV)
+        ENV_KEYS.each do |key|
+          value = env[key].to_s.strip
+          return value unless value.empty?
+        end
+        root = root_directory.to_s
+        return nil if root.empty?
+
+        revision_file(root) || git_head(root)
+      rescue StandardError
+        nil
+      end
+
+      private
+
+      def revision_file(root)
+        path = File.join(root, "REVISION")
+        return nil unless File.file?(path)
+
+        value = File.read(path, 200).to_s.strip
+        value.empty? ? nil : value
+      end
+
+      def git_head(root)
+        git = File.join(root, ".git")
+        # A worktree or submodule has a `.git` file pointing at the real dir.
+        if File.file?(git)
+          pointer = File.read(git, 1024).to_s[/\Agitdir:\s*(.+)$/, 1]
+          return nil unless pointer
+
+          git = File.expand_path(pointer.strip, root)
+        end
+        head_path = File.join(git, "HEAD")
+        return nil unless File.file?(head_path)
+
+        head = File.read(head_path, 1024).to_s.strip
+        ref = head[/\Aref:\s*(.+)\z/, 1]
+        return sha(head) unless ref
+
+        ref_path = File.join(git, ref)
+        return sha(File.read(ref_path, 200).to_s.strip) if File.file?(ref_path)
+
+        packed = File.join(git, "packed-refs")
+        return nil unless File.file?(packed)
+
+        line = File.foreach(packed).find { |entry| entry.strip.end_with?(" #{ref}") }
+        line ? sha(line.split(" ").first) : nil
+      end
+
+      def sha(value)
+        value.to_s.match?(/\A\h{7,64}\z/) ? value : nil
+      end
+    end
+  end
+end

data/lib/errorgap/sanitizer.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Errorgap
+  # Makes caller-supplied data safe to serialize: masks sensitive keys at any
+  # depth, cuts long strings and large or deeply nested collections, replaces
+  # invalid UTF-8 and breaks reference cycles. Without it one binary param or
+  # a self-referencing hash loses the whole notice.
+  class Sanitizer
+    FILTERED = "[FILTERED]"
+    TRUNCATED = "[TRUNCATED]"
+    CIRCULAR = "[CIRCULAR]"
+    MAX_STRING = 4096
+    MAX_ITEMS = 128
+    MAX_DEPTH = 8
+
+    # `filter_keys` entries are substrings (case-insensitive) or Regexps.
+    def initialize(filter_keys = [])
+      @filter_keys = Array(filter_keys).map { |key| key.is_a?(Regexp) ? key : key.to_s.downcase }
+    end
+
+    # Truncate only.
+    def clean(value)
+      walk(value, false, 0, {})
+    end
+
+    # Truncate and mask sensitive keys.
+    def filter(value)
+      walk(value, true, 0, {})
+    end
+
+    def self.string(value, limit = MAX_STRING)
+      text = value.to_s
+      text = text.dup.force_encoding(Encoding::UTF_8) unless text.encoding == Encoding::UTF_8
+      text = text.scrub("?") unless text.valid_encoding?
+      text.length > limit ? "#{text[0, limit]}#{TRUNCATED}" : text
+    rescue StandardError
+      "[UNPRINTABLE]"
+    end
+
+    private
+
+    def walk(value, masking, depth, seen)
+      case value
+      when Hash
+        nested(value, depth, seen) do
+          value.first(MAX_ITEMS).each_with_object({}) do |(key, item), result|
+            name = self.class.string(key, 256)
+            result[name] = masking && sensitive?(name) ? FILTERED : walk(item, masking, depth + 1, seen)
+          end
+        end
+      when Array
+        nested(value, depth, seen) do
+          value.first(MAX_ITEMS).map { |item| walk(item, masking, depth + 1, seen) }
+        end
+      when String, Symbol
+        self.class.string(value)
+      when Integer, NilClass, TrueClass, FalseClass
+        value
+      when Float
+        value.finite? ? value : value.to_s
+      else
+        if value.respond_to?(:to_hash)
+          walk(value.to_hash, masking, depth, seen)
+        else
+          self.class.string(value)
+        end
+      end
+    rescue StandardError
+      "[UNPRINTABLE]"
+    end
+
+    def nested(value, depth, seen)
+      return TRUNCATED if depth >= MAX_DEPTH
+      return CIRCULAR if seen[value.object_id]
+
+      seen[value.object_id] = true
+      begin
+        yield
+      ensure
+        seen.delete(value.object_id)
+      end
+    end
+
+    def sensitive?(key)
+      lowered = key.downcase
+      @filter_keys.any? do |filter|
+        filter.is_a?(Regexp) ? filter.match?(key) : lowered.include?(filter)
+      end
+    end
+  end
+end

data/lib/errorgap/sign_ins.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Errorgap
+  # Sign-ins to this app, shown beside SSH logins in Security › Logins.
+  # Opt in with `config.auth_events = true`. Only who, from where and the
+  # result are sent: never passwords, tokens or session ids.
+  class SignIns
+    OUTCOMES = %w[success failure password_reset mfa_failure locked].freeze
+
+    def initialize(configuration)
+      configure(configuration)
+    end
+
+    def configure(configuration)
+      @configuration = configuration
+    end
+
+    # `request` (Rack or ActionDispatch) fills in the IP, user agent and path.
+    def record(outcome, user: nil, request: nil, ip: nil, user_agent: nil, path: nil, method: nil,
+               occurred_at: nil, sync: false)
+      return unless @configuration.auth_events
+      return if @configuration.ignored_environment?
+
+      outcome = outcome.to_s
+      unless OUTCOMES.include?(outcome)
+        @configuration.logger&.warn("[errorgap] unknown sign-in outcome #{outcome.inspect}")
+        return
+      end
+
+      if request
+        ip ||= request.respond_to?(:remote_ip) ? request.remote_ip : request.ip
+        user_agent ||= request.user_agent
+        path ||= "#{request.request_method} #{request.path}"
+      end
+      event = { occurred_at: (occurred_at || Time.now).utc.iso8601(3), outcome: outcome }
+      event[:user] = user.to_s if present?(user)
+      event[:ip] = ip.to_s if present?(ip)
+      event[:user_agent] = user_agent.to_s[0, 512] if present?(user_agent)
+      event[:path] = path.to_s[0, 200] if present?(path)
+      event[:method] = method.to_s if present?(method)
+      payload = {
+        app: @configuration.app_name || @configuration.project_slug,
+        environment: @configuration.environment,
+        sdk: "errorgap-ruby #{Errorgap::VERSION}",
+        events: [event]
+      }
+
+      if sync || !@configuration.async
+        deliver(payload)
+      else
+        Errorgap.delivery.enqueue("logins/web", payload)
+        nil
+      end
+    end
+
+    def deliver(payload)
+      Errorgap.delivery.post("logins/web", payload)
+    end
+
+    private
+
+    def present?(value)
+      !value.nil? && !value.to_s.strip.empty?
+    end
+  end
+end

data/lib/errorgap/transacter.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-require "json"
-require "net/http"
-require "uri"
-
 module Errorgap
   class Transacter
     def initialize(configuration)
@@ -18,25 +14,11 @@ module Errorgap
       return if @configuration.ignored_environment?
       return unless should_sample?
 
-      Errorgap.register_thread(Thread.new { deliver(transaction) })
+      Errorgap.delivery.enqueue("transactions", transaction.to_h)
     end
 
     def deliver(transaction)
-      uri = URI.join(
-        @configuration.endpoint.end_with?("/") ? @configuration.endpoint : "#{@configuration.endpoint}/",
-        "api/projects/#{@configuration.project_slug}/transactions"
-      )
-      request = Net::HTTP::Post.new(uri)
-      request["Content-Type"] = "application/json"
-      request["User-Agent"] = "errorgap-ruby/#{Errorgap::VERSION}"
-      request["X-Errorgap-Project-Key"] = @configuration.api_key if present?(@configuration.api_key)
-      request.body = JSON.generate(transaction.to_h)
-
-      Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == "https") do |http|
-        http.request(request)
-      end
-    rescue StandardError => exception
-      @configuration.logger&.warn("[errorgap] APM delivery error: #{exception.class}: #{exception.message}")
+      Errorgap.delivery.post("transactions", transaction.to_h)
     end
 
     private
@@ -48,9 +30,5 @@ module Errorgap
 
       rand < rate
     end
-
-    def present?(value)
-      !value.nil? && !value.to_s.empty?
-    end
   end
 end

data/lib/errorgap/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Errorgap
-  VERSION = "0.7.0"
+  VERSION = "0.9.0"
 end

data/lib/errorgap/warden_hooks.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Errorgap
+  # Devise and other Warden apps: report sign-ins without touching the
+  # controllers. The Rails railtie installs these when `auth_events` is on;
+  # elsewhere call `Errorgap::WardenHooks.install` after configuring.
+  module WardenHooks
+    # Failures that are not a sign-in attempt: visiting a protected page
+    # signed out, an expired session, an account that is not confirmed.
+    NOT_ATTEMPTS = %i[unauthenticated timeout inactive unconfirmed].freeze
+    # Form fields that hold the name being signed in with.
+    USER_PARAMS = %w[email login username].freeze
+
+    class << self
+      def install
+        return false unless defined?(::Warden::Manager)
+        return true if @installed
+
+        ::Warden::Manager.after_authentication do |user, auth, opts|
+          Errorgap::WardenHooks.on_success(user, auth, opts)
+        end
+        ::Warden::Manager.before_failure do |env, opts|
+          Errorgap::WardenHooks.on_failure(env, opts)
+        end
+        @installed = true
+      end
+
+      def on_success(user, auth, _opts = {})
+        # A remember-me cookie signing the user back in is not a new sign-in.
+        return if auth.respond_to?(:winning_strategy) && auth.winning_strategy.respond_to?(:key) &&
+                  auth.winning_strategy.key == :rememberable
+
+        Errorgap.sign_in("success", user: identify(user), request: request_for(auth.env))
+      rescue StandardError => exception
+        Errorgap.configuration.logger&.warn("[errorgap] sign-in hook error: #{exception.class}: #{exception.message}")
+      end
+
+      def on_failure(env, opts = {})
+        message = opts[:message]&.to_sym
+        return if NOT_ATTEMPTS.include?(message)
+
+        request = request_for(env)
+        return unless request&.post?
+
+        outcome = message == :locked ? "locked" : "failure"
+        # Warden points PATH_INFO at the failure app before this runs; the
+        # form's path is in `attempted_path` (query string dropped).
+        path = opts[:attempted_path] ? "#{request.request_method} #{opts[:attempted_path].to_s.split('?').first}" : nil
+        Errorgap.sign_in(outcome, user: attempted_user(request, opts[:scope]), request: request, path: path)
+      rescue StandardError => exception
+        Errorgap.configuration.logger&.warn("[errorgap] sign-in hook error: #{exception.class}: #{exception.message}")
+      end
+
+      # How a signed-in user is named: `config.auth_user` (a method name or a
+      # proc), else email, username, login, then id.
+      def identify(user)
+        pick = Errorgap.configuration.auth_user
+        return pick.call(user) if pick.respond_to?(:call)
+        return user.public_send(pick) if pick && user.respond_to?(pick)
+
+        %i[email username login id].each do |name|
+          return user.public_send(name) if user.respond_to?(name)
+        end
+        nil
+      end
+
+      private
+
+      def attempted_user(request, scope)
+        params = request.params
+        fields = scope && params[scope.to_s].is_a?(Hash) ? params[scope.to_s] : params
+        USER_PARAMS.each do |name|
+          value = fields[name]
+          return value if value.is_a?(String) && !value.strip.empty?
+        end
+        nil
+      end
+
+      def request_for(env)
+        if defined?(::ActionDispatch::Request)
+          ::ActionDispatch::Request.new(env)
+        elsif defined?(::Rack::Request)
+          ::Rack::Request.new(env)
+        end
+      end
+    end
+  end
+end