goodsorcery 0.1.0

data/lib/goodsorcery/integrations/sidekiq/job_details.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Integrations
+    module Sidekiq
+      # The "job" section, transaction and queue tag for one Sidekiq job,
+      # read from the job hash when an error is reported. Arguments are left
+      # out when send_job_arguments is false or the job class has
+      # `sidekiq_options goodsorcery_arguments: false`.
+      class JobDetails
+        def initialize(job, queue: nil, worker: nil)
+          @job = job
+          @queue = queue || job["queue"]
+          @worker = worker
+        end
+
+        def open(scope)
+          scope.transaction = job_class
+          scope.merge_tags("queue" => @queue) if @queue
+          scope.enricher = self
+        end
+
+        def enrich(scope)
+          scope.job = to_h
+        end
+
+        def to_h
+          section = {
+            "framework" => "sidekiq", "class" => job_class, "id" => @job["jid"], "queue" => @queue,
+            "retry_count" => @job["retry_count"], "enqueued_at" => Event::Fields.iso_time(@job["enqueued_at"])
+          }
+          section["arguments"] = JobArguments.new.serialize(Array(@job["args"])) if send_arguments?
+          section.compact
+        end
+
+        private
+
+        def job_class
+          @job["class"].to_s
+        end
+
+        def send_arguments?
+          GoodSorcery.configuration.send_job_arguments && @job["goodsorcery_arguments"] != false &&
+            class_option != false
+        end
+
+        def class_option
+          klass = @worker&.class
+          klass.respond_to?(:get_sidekiq_options) ? klass.get_sidekiq_options["goodsorcery_arguments"] : nil
+        end
+      end
+    end
+  end
+end

data/lib/goodsorcery/integrations/sidekiq.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative "../integrations"
+require_relative "sidekiq/job_details"
+
+module GoodSorcery
+  module Integrations
+    # Sidekiq (DESIGN.md 5.1), without depending on the gem: a server
+    # middleware opens a scope per job and reports what escapes it, and an
+    # error handler catches anything raised outside the middleware. Each
+    # exception object is reported once. Jobs wrapped by Active Job are left
+    # to the Active Job layer.
+    module Sidekiq
+      def self.install
+        return if @installed
+
+        @installed = true
+        ::Sidekiq.configure_server do |config|
+          config.server_middleware { |chain| chain.add(ServerMiddleware) }
+          config.error_handlers << ErrorHandler.new.method(:report)
+        end
+      end
+
+      def self.active_job_wrapper?(job)
+        job.is_a?(Hash) && !job["wrapped"].nil?
+      end
+
+      # Sidekiq server middleware.
+      class ServerMiddleware
+        attr_accessor :config
+
+        def call(worker, job, queue, &)
+          return yield if Sidekiq.active_job_wrapper?(job)
+
+          Integrations.run_unit_of_work(JobDetails.new(job, queue: queue, worker: worker), &)
+        end
+      end
+
+      # Sidekiq error handler, registered as its #report method: Sidekiq
+      # passes (exception, context) before Sidekiq 7, (exception, context,
+      # config) since.
+      class ErrorHandler
+        def report(exception, context, _config = nil)
+          return if ReportedMark.reported?(exception)
+
+          job = context.is_a?(Hash) ? context[:job] : nil
+          GoodSorcery.with_scope do |scope|
+            JobDetails.new(job).open(scope) if job.is_a?(Hash) && !Sidekiq.active_job_wrapper?(job)
+            GoodSorcery.notify(exception)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/goodsorcery/integrations.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  # Framework integrations (DESIGN.md 5). Each layer opens a fresh scope per
+  # unit of work and leaves an enricher that reads its details only when an
+  # error is reported.
+  module Integrations
+    # GoodSorcery::DeliveryJob sends events; errors it raises are never
+    # reported (no recursion).
+    def self.delivery_job?(object)
+      GoodSorcery.const_defined?(:DeliveryJob, false) && object.is_a?(GoodSorcery::DeliveryJob)
+    end
+
+    # Runs the block in a fresh scope prepared by +details+ (an object with
+    # +open(scope)+), reporting any exception that escapes it, then
+    # re-raising.
+    def self.run_unit_of_work(details)
+      GoodSorcery.with_scope do |scope|
+        details.open(scope)
+        yield
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        GoodSorcery.notify(e)
+        raise
+      end
+    end
+  end
+end

data/lib/goodsorcery/job_arguments.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  # Turns job arguments (and other app values headed for an event) into JSON
+  # data the way a job system would see them, without loading anything or
+  # calling #inspect: GlobalID objects become their gid string, hash keys are
+  # kept (so key filtering applies), symbols become strings, times ISO 8601,
+  # and any other object a "#<ClassName>" placeholder. Depth, collection sizes
+  # and string lengths are capped like request params.
+  class JobArguments
+    MAX_DEPTH = 6
+    MAX_ITEMS = 50
+    MAX_KEYS = 100
+    MAX_STRING = 4_096
+    MAX_KEY_LENGTH = 200
+
+    def serialize(value, depth = 1)
+      case value
+      when Hash then serialize_hash(value, depth)
+      when Array then serialize_array(value, depth)
+      when String then Text.limit(value, MAX_STRING)
+      when Symbol then Text.limit(value.name, MAX_STRING)
+      when Integer, true, false, nil then value
+      else serialize_object(value)
+      end
+    end
+
+    private
+
+    def serialize_hash(hash, depth)
+      return Sanitizer::TRUNCATED if depth > MAX_DEPTH
+
+      hash.first(MAX_KEYS).to_h do |key, item|
+        [Text.limit(key_name(key), MAX_KEY_LENGTH), serialize(item, depth + 1)]
+      end
+    end
+
+    def serialize_array(array, depth)
+      return Sanitizer::TRUNCATED if depth > MAX_DEPTH
+
+      array.first(MAX_ITEMS).map { |item| serialize(item, depth + 1) }
+    end
+
+    def key_name(key)
+      plain_key?(key) ? key.to_s : placeholder(key)
+    end
+
+    def plain_key?(key)
+      key.is_a?(String) || key.is_a?(Symbol) || key.is_a?(Integer)
+    end
+
+    def serialize_object(value)
+      return value if value.is_a?(Float) && value.finite?
+      return value.to_s if value.is_a?(Numeric)
+      return value.name.to_s if value.is_a?(Module)
+
+      identify(value)
+    rescue StandardError
+      placeholder(value)
+    end
+
+    # Records as their GlobalID, times and dates as ISO 8601, anything else
+    # as a placeholder.
+    def identify(value)
+      return value.to_global_id.to_s if value.respond_to?(:to_global_id)
+      return Text.limit(value.iso8601, 64) if value.respond_to?(:iso8601)
+
+      placeholder(value)
+    end
+
+    def placeholder(value)
+      "#<#{value.class.name || "Object"}>"
+    rescue StandardError
+      "#<Object>"
+    end
+  end
+end

data/lib/goodsorcery/key_filter.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  # Masks values under sensitive keys, at any depth.
+  #
+  # Strings and symbols match a whole key, case-insensitively, with "-" and
+  # "_" treated as equal ("secret_key" matches "Secret-Key" but not
+  # "my_secret_key"). Regexps match any part of a key, in its own spelling
+  # or with "-" read as "_" (so /_key/ catches the "X-Access-Key" header).
+  # Patterns are compiled once, when the filter is built.
+  class KeyFilter
+    FILTERED = "[FILTERED]"
+    MAX_DEPTH = 16
+
+    def initialize(patterns)
+      regexps, names = Array(patterns).partition { |pattern| pattern.is_a?(Regexp) }
+      @names = names.to_h { |name| [normalize(name), true] }
+      @pattern = regexps.empty? ? nil : Regexp.union(regexps)
+    end
+
+    def filter_key?(key)
+      name = key.to_s
+      @names.key?(normalize(name)) || pattern_match?(name)
+    end
+
+    # Returns a filtered copy of +value+ (hashes and arrays are rebuilt).
+    def filter(value, depth = 0)
+      case value
+      when Hash then filter_hash(value, depth)
+      when Array then value.map { |item| filter(item, depth + 1) }
+      else value
+      end
+    end
+
+    # Masks sensitive parameters in a URL's query string ("user[password]=x"
+    # is matched on "password" as well). +extra+ (an ExtraFilterProbe) also
+    # applies the extra filters, such as Rails' filter_parameters.
+    def filter_url(url, extra = nil)
+      return url unless url.is_a?(String) && url.include?("?")
+
+      base, query = url.split("?", 2)
+      query, fragment = query.split("#", 2)
+      filtered = "#{base}?#{filter_query(query, extra)}"
+      fragment ? "#{filtered}##{fragment}" : filtered
+    end
+
+    private
+
+    def filter_hash(hash, depth)
+      return Sanitizer::TRUNCATED if depth > MAX_DEPTH
+
+      hash.each_with_object({}) do |(key, value), out|
+        filter_entry(out, key, value, depth)
+      end
+    end
+
+    def filter_entry(out, key, value, depth)
+      return out[key] = filter(value, depth + 1) unless filter_key?(key)
+
+      out[key] = FILTERED
+    end
+
+    def filter_query(query, extra)
+      query.split("&").map { |pair| filter_query_pair(pair, extra) }.join("&")
+    end
+
+    def filter_query_pair(pair, extra)
+      key = pair.split("=", 2).first.to_s
+      return pair unless sensitive_query_key?(key, extra)
+
+      "#{key}=#{FILTERED}"
+    end
+
+    def sensitive_query_key?(key, extra)
+      decoded = decode(key)
+      parts = decoded.scan(/[^\[\]]+/)
+      return true if [decoded, *parts].any? { |name| filter_key?(name) }
+
+      extra ? extra.filters_path?(parts) : false
+    end
+
+    def decode(key)
+      URI.decode_www_form_component(key)
+    rescue ArgumentError
+      key
+    end
+
+    def pattern_match?(name)
+      return false unless @pattern
+      return true if @pattern.match?(name)
+
+      name.include?("-") && @pattern.match?(name.tr("-", "_"))
+    end
+
+    def normalize(name)
+      name.to_s.downcase.tr("-", "_")
+    end
+  end
+end

data/lib/goodsorcery/log.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "logger"
+
+module GoodSorcery
+  # Prefixes and rate-limits the gem's own log lines. Never pass captured
+  # error data or API keys to it: messages name classes, counts and statuses.
+  class Log
+    PREFIX = "[goodsorcery]"
+
+    def initialize(&logger_source)
+      @logger_source = logger_source
+      @mutex = Mutex.new
+      @last_logged = {}
+    end
+
+    def debug(message) = write(:debug, message)
+    def info(message) = write(:info, message)
+    def warn(message) = write(:warn, message)
+    def error(message) = write(:error, message)
+
+    # Logs +message+ only the first time +key+ is seen in this process.
+    def warn_once(key, message)
+      warn(message) if first_time?(key, Float::INFINITY)
+    end
+
+    # Logs +message+ at most once per +interval+ seconds for +key+.
+    def warn_every(key, interval, message)
+      warn(message) if first_time?(key, interval)
+    end
+
+    def forget(key)
+      @mutex.synchronize { @last_logged.delete(key) }
+    end
+
+    private
+
+    def first_time?(key, interval)
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      @mutex.synchronize do
+        last = @last_logged[key]
+        next false if last && (interval == Float::INFINITY || now - last < interval)
+
+        @last_logged[key] = now
+        true
+      end
+    end
+
+    def write(level, message)
+      @logger_source.call&.public_send(level, "#{PREFIX} #{message}")
+    rescue StandardError
+      nil
+    end
+  end
+end

data/lib/goodsorcery/masked_key.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  # Shows an API key without revealing it: "gs_test_9f2c…e81a" becomes "gs_test_…e81a".
+  class MaskedKey
+    PREFIX = /\A(gs_[a-z]+_)/
+    MIN_LENGTH_FOR_TAIL = 16
+
+    def initialize(key)
+      @key = key.to_s
+    end
+
+    def to_s
+      return "(not set)" if @key.empty?
+
+      "#{prefix}…#{tail}"
+    end
+
+    private
+
+    def prefix
+      @key[PREFIX, 1].to_s
+    end
+
+    def tail
+      @key.length >= MIN_LENGTH_FOR_TAIL ? @key[-4..] : ""
+    end
+  end
+end

data/lib/goodsorcery/rack/middleware.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative "request_details"
+require_relative "request_enricher"
+
+module GoodSorcery
+  module Rack
+    # Rack middleware for Rails, Sinatra and plain Rack apps. Opens a fresh
+    # scope per request (cleared afterwards, whatever happens), records a
+    # "METHOD /path" breadcrumb and reports exceptions that propagate, then
+    # re-raises them. Exceptions an inner layer turned into a 5xx response
+    # (env["action_dispatch.exception"], env["sinatra.error"]) are reported
+    # too. The request itself is only read if an error is reported.
+    #
+    #   use GoodSorcery::Rack::Middleware
+    class Middleware
+      RESCUED_ERROR_KEYS = %w[action_dispatch.exception sinatra.error].freeze
+
+      def initialize(app, enricher: RequestEnricher)
+        @app = app
+        @enricher = enricher
+      end
+
+      def call(env)
+        GoodSorcery.with_scope do |scope|
+          open_request(scope, env)
+          response = call_app(env)
+          report_rescued_error(env, response)
+          response
+        end
+      end
+
+      private
+
+      def open_request(scope, env)
+        scope.request_env = env
+        scope.enricher = @enricher.new(env)
+        path = "#{env["SCRIPT_NAME"]}#{env["PATH_INFO"]}"
+        GoodSorcery.add_breadcrumb("#{env["REQUEST_METHOD"]} #{path}", category: "request")
+      end
+
+      def call_app(env)
+        @app.call(env)
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        GoodSorcery.notify(e)
+        raise
+      end
+
+      def report_rescued_error(env, response)
+        return unless response.is_a?(Array) && response.first.to_i >= 500
+
+        error = RESCUED_ERROR_KEYS.lazy.map { |key| env[key] }.find { |value| value.is_a?(Exception) }
+        GoodSorcery.notify(error) if error
+      end
+    end
+  end
+end

data/lib/goodsorcery/rack/request_details.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Rack
+    # Reads the "request" section from a Rack env, only when an error is
+    # reported. Uses what Rails (or Rack) already parsed and never reads the
+    # request body or loads the session. Filtering happens later, in
+    # EventFilter, like for every other section.
+    class RequestDetails
+      HEADER_PREFIX = "HTTP_"
+      DEFAULT_PORTS = { "http" => ":80", "https" => ":443" }.freeze
+      CONTENT_HEADERS = { "CONTENT_TYPE" => "Content-Type", "CONTENT_LENGTH" => "Content-Length" }.freeze
+
+      def initialize(env, config)
+        @env = env
+        @config = config
+      end
+
+      def to_h
+        {
+          "url" => url, "method" => @env["REQUEST_METHOD"], "format" => format,
+          "request_id" => request_id, "remote_ip" => remote_ip,
+          "params" => params, "headers" => headers, "session" => session
+        }.compact
+      end
+
+      def url
+        scheme = @env["rack.url_scheme"] || "http"
+        query = @env["QUERY_STRING"].to_s
+        base = "#{scheme}://#{host(scheme)}#{@env["SCRIPT_NAME"]}#{@env["PATH_INFO"]}"
+        query.empty? ? base : "#{base}?#{query}"
+      end
+
+      def request_id
+        @env["action_dispatch.request_id"] || @env["HTTP_X_REQUEST_ID"]
+      end
+
+      def remote_ip
+        rails_ip = @env["action_dispatch.remote_ip"]
+        (rails_ip && Text.safe_to_s(rails_ip)) || @env["REMOTE_ADDR"]
+      end
+
+      def headers
+        return nil unless @config.send_request_headers
+
+        @env.each_with_object({}) do |(name, value), headers|
+          header = header_name(name)
+          headers[header] = value if header && value.is_a?(String)
+        end
+      end
+
+      def params
+        return nil unless @config.send_request_params
+
+        parsed = @env["action_dispatch.request.parameters"]
+        parsed.is_a?(Hash) ? parsed : RackParams.new(@env).to_h
+      end
+
+      def session
+        return nil unless @config.send_session
+
+        session = @env["rack.session"]
+        return session if session.is_a?(Hash)
+        return nil unless session.respond_to?(:loaded?) && session.loaded?
+
+        session.to_hash
+      rescue StandardError
+        nil
+      end
+
+      private
+
+      def host(scheme)
+        host = @env["HTTP_HOST"] || [@env["SERVER_NAME"], @env["SERVER_PORT"]].compact.join(":")
+        host.delete_suffix(DEFAULT_PORTS.fetch(scheme, ""))
+      end
+
+      def format
+        formats = @env["action_dispatch.request.formats"]
+        formats.is_a?(Array) && formats.first ? Text.safe_to_s(formats.first) : nil
+      end
+
+      def header_name(name)
+        return CONTENT_HEADERS[name] unless name.start_with?(HEADER_PREFIX)
+
+        name.delete_prefix(HEADER_PREFIX).split("_").map(&:capitalize).join("-")
+      end
+    end
+
+    # Params a plain Rack app already has: the parsed query string, plus the
+    # form body only if the app parsed it already (Rack caches it in the env).
+    # Path params from the Rails router are added when present.
+    class RackParams
+      def initialize(env)
+        @env = env
+      end
+
+      def to_h
+        merged = query.merge(cached_form)
+        path = @env["action_dispatch.request.path_parameters"]
+        path.is_a?(Hash) ? merged.merge(path.transform_keys(&:to_s)) : merged
+      rescue StandardError
+        nil
+      end
+
+      private
+
+      def query
+        return {} unless defined?(::Rack::Request)
+
+        ::Rack::Request.new(@env).GET
+      end
+
+      def cached_form
+        form = @env["rack.request.form_hash"]
+        form.is_a?(Hash) ? form : {}
+      end
+    end
+  end
+end

data/lib/goodsorcery/rack/request_enricher.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  module Rack
+    # Fills in the request section when an error is reported during a Rack
+    # request. Leaves it to config.request_extractor when one is configured.
+    class RequestEnricher
+      def initialize(env)
+        @env = env
+      end
+
+      def enrich(scope)
+        config = GoodSorcery.configuration
+        return if config.request_extractor || scope.request
+
+        scope.request = RequestDetails.new(@env, config).to_h
+      end
+    end
+  end
+end

data/lib/goodsorcery/rails_filter_parameters.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  # Rails.application.config.filter_parameters as an extra filter, using
+  # ActiveSupport::ParameterFilter so Rails' own matching rules apply. Built
+  # whenever the event pipeline is compiled; nil outside a Rails app.
+  module RailsFilterParameters
+    def self.build
+      filters = rails_filters
+      return nil if filters.nil? || filters.empty?
+
+      ::ActiveSupport::ParameterFilter.new(filters, mask: KeyFilter::FILTERED)
+    rescue StandardError
+      nil
+    end
+
+    def self.rails_filters
+      return nil unless defined?(::ActiveSupport::ParameterFilter) && defined?(::Rails)
+      return nil unless ::Rails.respond_to?(:application) && ::Rails.application
+
+      ::Rails.application.config.filter_parameters
+    end
+  end
+end

data/lib/goodsorcery/railtie.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative "../goodsorcery" unless defined?(GoodSorcery::Client)
+require_relative "integrations/rails"
+
+module GoodSorcery
+  # Rails integration (DESIGN.md 5): defaults from Rails (root, environment,
+  # logger, filter_parameters and the REVISION file are read lazily by
+  # Configuration), the Rack middleware after ActionDispatch::ShowExceptions,
+  # the Rails.error subscriber, the Active Job, Action Cable and notification
+  # breadcrumb layers, rake tasks and the goodsorcery:install generator.
+  class Railtie < ::Rails::Railtie
+    initializer "goodsorcery.middleware" do |app|
+      GoodSorcery::Integrations::Rails.install_middleware(app.config.middleware)
+    end
+
+    initializer "goodsorcery.framework_hooks" do
+      GoodSorcery::Integrations::Rails.install_framework_hooks
+    end
+
+    config.after_initialize do
+      GoodSorcery::Integrations::Rails.after_initialize
+    end
+
+    rake_tasks do
+      require_relative "tasks"
+    end
+  end
+end

data/lib/goodsorcery/release_detector.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  # Finds the running release without shelling out to git: the first present
+  # deploy environment variable, else the Capistrano REVISION file.
+  class ReleaseDetector
+    ENV_NAMES = %w[
+      GOODSORCERY_RELEASE KAMAL_VERSION HEROKU_SLUG_COMMIT HEROKU_BUILD_COMMIT
+      RENDER_GIT_COMMIT GIT_COMMIT GIT_SHA SOURCE_VERSION
+    ].freeze
+    MAX_LENGTH = 64
+
+    def initialize(root)
+      @root = root
+    end
+
+    def release
+      value = release_from_env || release_from_revision_file
+      value&.strip&.slice(0, MAX_LENGTH)
+    end
+
+    private
+
+    def release_from_env
+      ENV_NAMES.each do |name|
+        value = EnvFlag.present(name)
+        return value if value
+      end
+      nil
+    end
+
+    def release_from_revision_file
+      return unless @root
+
+      path = File.join(@root.to_s, "REVISION")
+      content = File.file?(path) ? File.read(path, 256).to_s.strip : ""
+      content.empty? ? nil : content
+    rescue SystemCallError
+      nil
+    end
+  end
+end