goodsorcery 0.1.0

data/lib/goodsorcery/event/builder.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Event
+    # Builds the event hash for one exception in the caller's thread, already
+    # within the server's size limits (INGEST_API.md). Filtering and
+    # before_send happen afterwards, in EventPipeline.
+    class Builder
+      NOTIFIER = { "name" => NOTIFIER_NAME, "version" => VERSION }.freeze
+      MAX_TRANSACTION_LENGTH = 1_000
+
+      def initialize(config)
+        @config = config
+        @sanitizer = Sanitizer.new
+        @exception_section = build_exception_section(config)
+        @request_section = RequestSection.new(config.request_extractor, @sanitizer)
+        @runtime = RuntimeContext.new
+      end
+
+      def build(exception, options, scope)
+        event = header(options)
+        event["exception"] = @exception_section.build(exception)
+        event.merge!(scope_sections(options, scope))
+        event["context"] = context(exception, options, scope)
+        event["fingerprint"] = Fields.fingerprint(options[:fingerprint])
+        event["notifier"] = NOTIFIER.dup
+        event.compact
+      end
+
+      private
+
+      def build_exception_section(config)
+        paths = Backtrace::Paths.new(config.root)
+        code_context = Backtrace::CodeContext.new(Backtrace::LineCache.new(paths))
+        ExceptionSection.new(paths: paths, code_context: code_context, send_code_context: config.send_code_context)
+      end
+
+      def header(options)
+        {
+          "event_id" => Fields.event_id,
+          "occurred_at" => Fields.timestamp,
+          "level" => Fields.level(options[:level]),
+          "release" => @config.release
+        }
+      end
+
+      def scope_sections(options, scope)
+        {
+          "transaction" => Fields.text(options[:transaction] || scope.transaction, MAX_TRANSACTION_LENGTH),
+          "request" => @request_section.build(scope),
+          "job" => JobSection.build(scope.job, @sanitizer),
+          "user" => UserSection.build(options[:user] || scope.user),
+          "breadcrumbs" => breadcrumbs(scope),
+          "tags" => presence(TagSet.merge(@config.tags, scope.tags, options[:tags]))
+        }
+      end
+
+      def presence(hash)
+        hash.empty? ? nil : hash
+      end
+
+      def breadcrumbs(scope)
+        crumbs = scope.breadcrumbs.to_a.last(100)
+        crumbs.empty? ? nil : crumbs.map(&:dup)
+      end
+
+      def context(exception, options, scope)
+        context = @runtime.to_h
+        causes = CauseChain.new(exception).to_a
+        context["cause"] = causes unless causes.empty?
+        context.merge!(@sanitizer.sanitize(scope.context)) if scope.context.any?
+        context.merge!(@sanitizer.sanitize(options[:context])) if options[:context].is_a?(Hash)
+        context
+      end
+    end
+  end
+end

data/lib/goodsorcery/event/cause_chain.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Event
+    # Up to three causes of an exception, as class and message.
+    class CauseChain
+      MAX_CAUSES = 3
+      MAX_MESSAGE_LENGTH = 1_000
+
+      def initialize(exception)
+        @exception = exception
+      end
+
+      def to_a
+        causes.map do |cause|
+          {
+            "class" => ExceptionSection.class_name(cause),
+            "message" => ExceptionSection.message(cause, MAX_MESSAGE_LENGTH)
+          }
+        end
+      end
+
+      private
+
+      def causes
+        seen = {}.compare_by_identity
+        seen[@exception] = true
+        current = @exception.cause
+        list = []
+        while current && list.size < MAX_CAUSES && !seen[current]
+          seen[current] = true
+          list << current
+          current = current.cause
+        end
+        list
+      end
+    end
+  end
+end

data/lib/goodsorcery/event/exception_section.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Event
+    # The "exception" section: class, message and backtrace frames, with code
+    # context for the top in-app frames.
+    class ExceptionSection
+      MAX_CLASS_LENGTH = 255
+      MAX_MESSAGE_LENGTH = 10_000
+      MAX_FRAMES = 250
+      CONTEXT_FRAMES = 10
+
+      def initialize(paths:, code_context:, send_code_context:)
+        @paths = paths
+        @code_context = code_context
+        @send_code_context = send_code_context
+        @parser = Backtrace::Parser.new
+      end
+
+      def build(exception)
+        {
+          "class" => self.class.class_name(exception),
+          "message" => self.class.message(exception, MAX_MESSAGE_LENGTH),
+          "backtrace" => backtrace(exception)
+        }
+      end
+
+      def self.class_name(exception)
+        Text.limit(exception.class.name || exception.class.to_s, MAX_CLASS_LENGTH)
+      end
+
+      def self.message(exception, max)
+        Text.limit(exception.message, max)
+      rescue StandardError
+        ""
+      end
+
+      private
+
+      def backtrace(exception)
+        remaining = @send_code_context ? CONTEXT_FRAMES : 0
+        @parser.frames(exception, limit: MAX_FRAMES).map do |frame|
+          hash = frame_hash(frame)
+          next hash unless hash["in_app"] && remaining.positive?
+
+          remaining -= 1
+          add_code_context(hash, frame)
+        end
+      end
+
+      def frame_hash(frame)
+        path = frame.path.to_s
+        {
+          "file" => Text.limit(@paths.relative(path), 1_024),
+          "line" => frame.line,
+          "method" => frame.label && Text.limit(frame.label, 255),
+          "in_app" => @paths.in_app?(path)
+        }.compact
+      end
+
+      def add_code_context(hash, frame)
+        context = @code_context.around(frame.path.to_s, frame.line)
+        hash["context"] = context if context
+        hash
+      end
+    end
+  end
+end

data/lib/goodsorcery/event/fields.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Event
+    # Small normalizers for top-level event fields.
+    module Fields
+      LEVELS = %w[fatal error warning info].freeze
+      LEVEL_ALIASES = { "warn" => "warning", "critical" => "fatal", "debug" => "info" }.freeze
+      MAX_FINGERPRINT_PARTS = 10
+      MAX_FINGERPRINT_LENGTH = 200
+
+      module_function
+
+      def level(value, default = "error")
+        name = value.to_s.downcase
+        name = LEVEL_ALIASES.fetch(name, name)
+        LEVELS.include?(name) ? name : default
+      end
+
+      def fingerprint(value)
+        return nil if value.nil?
+
+        parts = Array(value).select { |part| part.is_a?(String) || part.is_a?(Symbol) || part.is_a?(Numeric) }
+        parts = parts.first(MAX_FINGERPRINT_PARTS).map { |part| Text.limit(part.to_s, MAX_FINGERPRINT_LENGTH) }
+        parts.empty? ? nil : parts
+      end
+
+      def event_id
+        SecureRandom.respond_to?(:uuid_v7) ? SecureRandom.uuid_v7 : SecureRandom.uuid
+      end
+
+      def timestamp(time = Time.now)
+        time.utc.iso8601(3)
+      end
+
+      # A job timestamp as ISO 8601: a Time (or Date), epoch seconds, epoch
+      # milliseconds (Sidekiq 8) or an already formatted string.
+      def iso_time(value)
+        case value
+        when nil then nil
+        when String then Text.limit(value, 64)
+        when Time then timestamp(value)
+        when Numeric then timestamp(Time.at(epoch_seconds(value)))
+        else formatted_time(value)
+        end
+      end
+
+      def epoch_seconds(value)
+        value > 100_000_000_000 ? value / 1000.0 : value
+      end
+
+      def formatted_time(value)
+        value.respond_to?(:iso8601) ? Text.limit(value.iso8601, 64) : nil
+      end
+
+      def text(value, max)
+        value.nil? ? nil : Text.limit(value, max)
+      end
+    end
+  end
+end

data/lib/goodsorcery/event/job_section.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Event
+    # The "job" section (DESIGN.md 5.1): which job ran, where, and with which
+    # arguments. Keeps the known fields only; arguments are capped like
+    # request params and filtered by EventFilter afterwards.
+    module JobSection
+      TEXT_FIELDS = %w[
+        framework class id provider_job_id queue mailer action enqueued_at scheduled_at
+      ].freeze
+      INTEGER_FIELDS = %w[priority executions retry_count].freeze
+      MAX_TEXT_LENGTH = 255
+
+      module_function
+
+      def build(raw, sanitizer)
+        return nil unless raw.is_a?(Hash)
+
+        section = {}
+        raw.each { |key, value| add(section, key.to_s, value, sanitizer) }
+        section.empty? ? nil : section
+      end
+
+      def add(section, key, value, sanitizer)
+        return if value.nil?
+
+        section[key] = Text.limit(value, MAX_TEXT_LENGTH) if TEXT_FIELDS.include?(key)
+        section[key] = value if INTEGER_FIELDS.include?(key) && value.is_a?(Integer)
+        section[key] = sanitizer.sanitize(value) if key == "arguments"
+      end
+    end
+  end
+end

data/lib/goodsorcery/event/request_section.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Event
+    # The "request" section. Integrations either store a ready hash in
+    # scope.request, or keep the Rack env in scope.request_env and set
+    # config.request_extractor (an object responding to +extract(env)+), so
+    # the request is only read when an error is actually reported.
+    class RequestSection
+      SCALARS = %w[url method format request_id remote_ip].freeze
+      NESTED = %w[params headers session].freeze
+
+      def initialize(extractor, sanitizer)
+        @extractor = extractor
+        @sanitizer = sanitizer
+      end
+
+      def build(scope)
+        raw = scope.request || extract(scope.request_env)
+        return nil unless raw.is_a?(Hash)
+
+        section = {}
+        raw.each { |key, value| add(section, key.to_s, value) }
+        section.empty? ? nil : section
+      end
+
+      private
+
+      def extract(env)
+        env && @extractor ? @extractor.extract(env) : nil
+      end
+
+      def add(section, key, value)
+        return if value.nil?
+
+        section[key] = Text.limit(value, 4_096) if SCALARS.include?(key)
+        section[key] = @sanitizer.sanitize(value) if NESTED.include?(key) && value.is_a?(Hash)
+      end
+    end
+  end
+end

data/lib/goodsorcery/event/runtime_context.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Event
+    # Runtime details for the Environment tab: Ruby (and YJIT), Rails, host,
+    # process and the RAILS_ENV/RACK_ENV variables only.
+    class RuntimeContext
+      ENV_NAMES = %w[RAILS_ENV RACK_ENV].freeze
+
+      def initialize
+        @ruby = ruby_label
+        @host = hostname
+      end
+
+      def to_h
+        {
+          "runtime" => { "ruby" => @ruby, "rails" => rails_version }.compact,
+          "host" => @host,
+          "process" => "pid #{Process.pid} · thread #{thread_label}",
+          "env" => env_vars
+        }.compact
+      end
+
+      private
+
+      def env_vars
+        vars = ENV.to_h.slice(*ENV_NAMES)
+        vars.empty? ? nil : vars
+      end
+
+      def ruby_label
+        yjit = defined?(::RubyVM::YJIT) && ::RubyVM::YJIT.respond_to?(:enabled?) && ::RubyVM::YJIT.enabled?
+        yjit ? "#{RUBY_VERSION} (YJIT on)" : RUBY_VERSION
+      end
+
+      def rails_version
+        ::Rails.version.to_s if defined?(::Rails) && ::Rails.respond_to?(:version)
+      end
+
+      def hostname
+        Text.limit(Socket.gethostname, 255)
+      rescue StandardError
+        nil
+      end
+
+      def thread_label
+        thread = Thread.current
+        thread.name || thread.native_thread_id || thread.object_id
+      end
+    end
+  end
+end

data/lib/goodsorcery/event/tag_set.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Event
+    # Merges tag hashes (later ones win) and keeps what the server accepts:
+    # up to 25 keys matching KEY with scalar values of at most 200 chars.
+    module TagSet
+      KEY = /\A[a-z0-9_.-]{1,32}\z/i
+      MAX_TAGS = 25
+      MAX_VALUE_LENGTH = 200
+
+      module_function
+
+      def merge(*hashes)
+        merged = {}
+        hashes.each { |hash| add(merged, hash) if hash.is_a?(Hash) }
+        merged.first(MAX_TAGS).to_h
+      end
+
+      def add(merged, hash)
+        hash.each do |key, raw|
+          name = key.to_s
+          value = scalar(raw)
+          merged[name] = value if KEY.match?(name) && !value.nil?
+        end
+      end
+
+      def scalar(raw)
+        case raw
+        when String, Symbol then Text.limit(raw.to_s, MAX_VALUE_LENGTH)
+        when Integer, true, false then raw
+        when Float then finite_float(raw)
+        end
+      end
+
+      def finite_float(raw)
+        raw.finite? ? raw : nil
+      end
+    end
+  end
+end

data/lib/goodsorcery/event/user_section.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Event
+    # Keeps only the four user fields the server accepts. Reads them from a
+    # hash (string or symbol keys) or from an object such as a User model.
+    module UserSection
+      FIELDS = %w[id email username name].freeze
+      MAX_LENGTH = 255
+
+      module_function
+
+      def build(user)
+        return nil if user.nil?
+
+        fields = FIELDS.to_h { |field| [field, value(read(user, field))] }.compact
+        fields.empty? ? nil : fields
+      end
+
+      def read(user, field)
+        return user.fetch(field) { user[field.to_sym] } if user.is_a?(Hash)
+
+        user.public_send(field) if user.respond_to?(field)
+      rescue StandardError
+        nil
+      end
+
+      def value(raw)
+        return raw if raw.is_a?(Integer)
+        return nil if raw.nil?
+
+        Text.limit(raw, MAX_LENGTH)
+      end
+    end
+  end
+end

data/lib/goodsorcery/event_filter.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  # Applies the key filter, and any extra filters, to every part of an event
+  # that holds free-form data: request params, headers, session and URL query,
+  # job arguments, context, tags and breadcrumb data. The user section is left alone on
+  # purpose (DESIGN.md 3.4).
+  #
+  # Extra filters (config.extra_filters) are objects responding to
+  # +filter(hash) -> hash+, for example ActiveSupport::ParameterFilter, which
+  # the Rails integration adds for Rails.application.config.filter_parameters.
+  class EventFilter
+    REQUEST_SECTIONS = %w[params headers session].freeze
+
+    def initialize(key_filter, extra_filters = [])
+      @key_filter = key_filter
+      @extra_filters = Array(extra_filters)
+      @extra_probe = @extra_filters.empty? ? nil : ExtraFilterProbe.new(@extra_filters)
+    end
+
+    def filter_event(event)
+      filter_request(event["request"]) if event["request"]
+      filter_job(event["job"]) if event["job"]
+      filter_section(event, "context")
+      filter_section(event, "tags")
+      event["breadcrumbs"]&.each { |crumb| filter_section(crumb, "data") }
+      event
+    end
+
+    def filter_hash(hash)
+      filtered = @key_filter.filter(hash)
+      @extra_filters.reduce(filtered) { |result, extra| extra.filter(result) }
+    end
+
+    private
+
+    def filter_request(request)
+      request["url"] = @key_filter.filter_url(request["url"], @extra_probe) if request["url"]
+      REQUEST_SECTIONS.each { |name| filter_section(request, name) }
+    end
+
+    # Arguments are often an array, so they are filtered inside a wrapper hash.
+    def filter_job(job)
+      return unless job.key?("arguments")
+
+      job["arguments"] = filter_hash({ "arguments" => job["arguments"] })["arguments"]
+    end
+
+    def filter_section(container, name)
+      value = container[name]
+      container[name] = filter_hash(value) if value.is_a?(Hash)
+    end
+  end
+end

data/lib/goodsorcery/event_pipeline.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  # Builds, filters and passes an event through before_send, in the caller's
+  # thread. Returns the event, or nil when before_send drops it (or raises).
+  class EventPipeline
+    def initialize(config, log)
+      @builder = Event::Builder.new(config)
+      @filter = EventFilter.new(KeyFilter.new(config.filter_params), config.event_filters)
+      @before_send = config.before_send
+      @log = log
+    end
+
+    def prepare(exception, options, scope)
+      event = @builder.build(exception, options, scope)
+      @filter.filter_event(event)
+      apply_before_send(event)
+    end
+
+    private
+
+    def apply_before_send(event)
+      return event unless @before_send
+
+      result = @before_send.call(event)
+      return result if result.is_a?(Hash)
+
+      @log.warn_once(:before_send_type, "before_send returned #{result.class}; event dropped") unless result.nil?
+      nil
+    rescue StandardError => e
+      @log.warn_every(:before_send_error, 60, "before_send raised #{e.class}; event dropped")
+      nil
+    end
+  end
+end

data/lib/goodsorcery/event_serializer.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  # Serializes an event to JSON, dropping (and logging, by size only) events
+  # over the server's 512 KiB limit.
+  class EventSerializer
+    MAX_BYTES = 524_288
+
+    def initialize(log)
+      @log = log
+    end
+
+    def dump(event)
+      json = JSON.generate(event)
+      return json if json.bytesize <= MAX_BYTES
+
+      @log.warn_once(:oversized_event,
+                     "dropped an event of #{json.bytesize} bytes (limit 512 KiB); later drops are not logged")
+      nil
+    rescue StandardError => e
+      @log.warn_every(:unserializable_event, 300, "dropped an event that could not be serialized (#{e.class})")
+      nil
+    end
+  end
+end

data/lib/goodsorcery/exception_coercion.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  # Turns what was passed to notify into an exception. A message becomes a
+  # RuntimeError whose backtrace is the caller's stack (without this gem's
+  # own frames).
+  module ExceptionCoercion
+    GEM_DIR = File.expand_path("..", __dir__)
+
+    def self.coerce(error)
+      return error if error.is_a?(Exception)
+
+      exception = RuntimeError.new(Text.safe_to_s(error))
+      exception.set_backtrace(caller.reject { |line| line.start_with?(GEM_DIR) })
+      exception
+    end
+  end
+end

data/lib/goodsorcery/extra_filter_probe.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  # Asks the extra filters (such as Rails' filter_parameters) whether they
+  # would filter a key path, by filtering a hash that holds one probe value
+  # at that path. Used for URL query parameters, which are text, not hashes.
+  class ExtraFilterProbe
+    PROBE = "goodsorcery-probe"
+
+    def initialize(extra_filters)
+      @extra_filters = Array(extra_filters)
+    end
+
+    # +parts+ is a parameter name split on brackets: "user[pin]" => ["user", "pin"].
+    def filters_path?(parts)
+      return false if parts.empty?
+
+      @extra_filters.any? { |extra| !survives?(extra.filter(probe(parts)), parts) }
+    end
+
+    private
+
+    def probe(parts)
+      parts.reverse.reduce(PROBE) { |inner, part| { part => inner } }
+    end
+
+    def survives?(value, parts)
+      parts.each { |part| value = value.is_a?(Hash) ? value[part] : nil }
+      value == PROBE
+    end
+  end
+end

data/lib/goodsorcery/ignore_list.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  # Exception classes that are never reported, by name; subclasses (any
+  # ancestor with a listed name) are ignored too.
+  class IgnoreList
+    def initialize(entries)
+      @names = Array(entries).to_h { |entry| [entry.is_a?(Module) ? entry.name : entry.to_s, true] }
+    end
+
+    def ignore?(exception)
+      exception.class.ancestors.any? { |ancestor| @names.key?(ancestor.name.to_s) }
+    end
+  end
+end