goodsorcery 0.1.0

data/lib/goodsorcery/transport.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+require_relative "transport/result"
+require_relative "transport/connection"
+require_relative "transport/batch_chunks"
+
+module GoodSorcery
+  # Talks to the ingest API over HTTP(S): keep-alive, gzip, Bearer auth.
+  # Takes serialized events (JSON strings) and returns Transport::Result
+  # objects. Logs statuses and counts only, never bodies or the full key.
+  class Transport
+    EVENTS_PATH = "/api/v1/events"
+    BATCH_PATH = "/api/v1/events/batch"
+    VERIFY_PATH = "/api/v1/verify"
+    GZIP_MIN_BYTES = 1_024
+
+    def initialize(endpoint:, api_key:, open_timeout:, read_timeout:, log:)
+      @uri = URI(endpoint)
+      @api_key = api_key.to_s
+      @timeouts = { open_timeout: open_timeout, read_timeout: read_timeout }
+      @log = log
+      @connection = Connection.new(@uri, **@timeouts)
+      @connection_lock = Mutex.new
+      @batch_supported = true
+      warn_about_plain_http
+    end
+
+    def post_event(json)
+      result = send_request(post(EVENTS_PATH, json))
+      log_result(result, 1)
+      result
+    end
+
+    # Sends events in as few requests as possible. On a 429 or a retryable
+    # failure it stops and returns that result with the unsent events in
+    # +pending+. Falls back to one request per event if the server has no
+    # batch endpoint (404), and remembers that. A batch the server (or a
+    # proxy in front of it) refuses as too large (413) is split in halves.
+    def post_batch(jsons)
+      return post_each(jsons) unless @batch_supported
+
+      chunks = BatchChunks.split(jsons)
+      chunks.each_with_index do |chunk, index|
+        result = post_chunk(chunk)
+        return result.with_pending(unsent(result, chunk, chunks[(index + 1)..])) if result.retry_later?
+        return post_each(chunks[index..].flatten) if result.not_found? && batch_unsupported!
+      end
+      Result.new(:success)
+    end
+
+    def verify
+      send_request(Net::HTTP::Get.new(path(VERIFY_PATH), headers))
+    end
+
+    def close
+      @connection_lock.synchronize { @connection.close }
+    end
+
+    def batch_supported?
+      @batch_supported
+    end
+
+    # Shows the host and the masked key only.
+    def inspect
+      "#<#{self.class.name} #{@uri.scheme}://#{@uri.host}:#{@uri.port} api_key=#{MaskedKey.new(@api_key)}>"
+    end
+
+    private
+
+    def post_chunk(chunk)
+      result = send_request(post(BATCH_PATH, BatchChunks.body(chunk)))
+      return post_halves(chunk) if result.status == 413 && chunk.size > 1
+
+      log_result(result, chunk.size) unless result.not_found?
+      log_partial_rejections(result) if result.success?
+      result
+    end
+
+    def post_halves(chunk)
+      halves = chunk.each_slice((chunk.size + 1) / 2).to_a
+      halves.each_with_index do |half, index|
+        result = post_chunk(half)
+        return result.with_pending(unsent(result, half, halves[(index + 1)..])) if result.retry_later?
+      end
+      Result.new(:success)
+    end
+
+    # What is left after a 429 or retryable result for +chunk+.
+    def unsent(result, chunk, later_chunks)
+      (result.pending.empty? ? chunk : result.pending) + later_chunks.flatten
+    end
+
+    def post_each(jsons)
+      jsons.each_with_index do |json, index|
+        result = post_event(json)
+        return result.with_pending(jsons[index..]) if result.retry_later?
+      end
+      Result.new(:success)
+    end
+
+    def batch_unsupported!
+      @log.info("the server has no batch endpoint; sending events one per request")
+      @batch_supported = false
+      true
+    end
+
+    def send_request(request)
+      Result.from_response(perform(request))
+    rescue StandardError => e
+      Result.from_error(e)
+    end
+
+    # The persistent connection serves one thread at a time; a concurrent
+    # caller (the :sync delivery in a busy app) gets a short-lived one.
+    def perform(request)
+      return Connection.new(@uri, **@timeouts).request_once(request) unless @connection_lock.try_lock
+
+      begin
+        @connection.request(request)
+      ensure
+        @connection_lock.unlock
+      end
+    end
+
+    def post(path_name, body)
+      request = Net::HTTP::Post.new(path(path_name), headers)
+      if body.bytesize >= GZIP_MIN_BYTES
+        request["Content-Encoding"] = "gzip"
+        body = Zlib.gzip(body, level: Zlib::BEST_SPEED)
+      end
+      request.body = body
+      request
+    end
+
+    def headers
+      {
+        "Authorization" => "Bearer #{@api_key}",
+        "Content-Type" => "application/json",
+        "Accept" => "application/json",
+        "User-Agent" => "#{NOTIFIER_NAME}/#{VERSION}"
+      }
+    end
+
+    def path(name)
+      "#{@uri.path.chomp("/")}#{name}"
+    end
+
+    def log_result(result, count)
+      return log_invalid_key if result.status == 401
+      return if result.success? || result.retry_later?
+
+      @log.warn_every(:"rejected_#{result.status}", 60, "the server rejected #{count} event(s) (#{result.description})")
+    end
+
+    def log_invalid_key
+      @log.warn_once(:invalid_api_key, "invalid API key #{MaskedKey.new(@api_key)} (HTTP 401); events are dropped")
+    end
+
+    def log_partial_rejections(result)
+      rejected = Array(result.body["rejected"])
+      return if rejected.empty?
+
+      @log.warn_every(:batch_rejections, 60, "the server rejected #{rejected.size} event(s) of a batch as invalid")
+    end
+
+    def warn_about_plain_http
+      return unless @uri.scheme == "http" && !%w[localhost 127.0.0.1 ::1].include?(@uri.host)
+
+      @log.warn_once(:plain_http, "endpoint #{@uri.host} uses plain HTTP; the API key is sent unencrypted")
+    end
+  end
+end

data/lib/goodsorcery/verification.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  # The outcome of GoodSorcery.verify: can this endpoint be reached, and does
+  # it accept the API key? Messages show the masked key only.
+  class Verification
+    attr_reader :message, :environment
+
+    NO_KEY = "no API key configured (set GOODSORCERY_API_KEY or config.api_key)"
+
+    def self.run(config, transport)
+      return new(false, NO_KEY) if config.api_key.to_s.empty?
+
+      from_result(transport.verify, config)
+    end
+
+    def self.from_result(result, config)
+      return success(result, config) if result.success?
+      return new(false, "the server rejected API key #{config.masked_api_key} (HTTP 401)") if result.status == 401
+      return new(false, "#{config.endpoint_label} has no verify endpoint (HTTP 404)") if result.not_found?
+      return new(false, "rate limited (HTTP 429); try again in #{result.retry_after} s") if result.rate_limited?
+
+      new(false, "could not reach #{config.endpoint_label} (#{result.description})")
+    end
+
+    def self.success(result, config)
+      environment = result.body["environment"]
+      label = environment ? " for environment #{environment.to_s.inspect}" : ""
+      message = "API key #{config.masked_api_key} is valid#{label} at #{config.endpoint_label}"
+      new(true, message, environment: environment)
+    end
+
+    def initialize(succeeded, message, environment: nil)
+      @ok = succeeded
+      @message = message
+      @environment = environment
+    end
+
+    def ok?
+      @ok
+    end
+
+    def to_s
+      message
+    end
+  end
+end

data/lib/goodsorcery/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module GoodSorcery
+  VERSION = "0.1.0"
+  NOTIFIER_NAME = "goodsorcery-ruby"
+end

data/lib/goodsorcery.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require "json"
+require "logger"
+require "monitor"
+require "net/http"
+require "securerandom"
+require "socket"
+require "time"
+require "uri"
+require "zlib"
+
+require_relative "goodsorcery/version"
+require_relative "goodsorcery/errors"
+require_relative "goodsorcery/log"
+require_relative "goodsorcery/defaults"
+require_relative "goodsorcery/env_flag"
+require_relative "goodsorcery/masked_key"
+require_relative "goodsorcery/release_detector"
+require_relative "goodsorcery/configuration"
+require_relative "goodsorcery/environment_gate"
+require_relative "goodsorcery/text"
+require_relative "goodsorcery/sanitizer"
+require_relative "goodsorcery/key_filter"
+require_relative "goodsorcery/extra_filter_probe"
+require_relative "goodsorcery/event_filter"
+require_relative "goodsorcery/rails_filter_parameters"
+require_relative "goodsorcery/job_arguments"
+require_relative "goodsorcery/scope"
+require_relative "goodsorcery/backtrace/parser"
+require_relative "goodsorcery/backtrace/paths"
+require_relative "goodsorcery/backtrace/line_cache"
+require_relative "goodsorcery/backtrace/code_context"
+require_relative "goodsorcery/event/fields"
+require_relative "goodsorcery/event/exception_section"
+require_relative "goodsorcery/event/cause_chain"
+require_relative "goodsorcery/event/runtime_context"
+require_relative "goodsorcery/event/user_section"
+require_relative "goodsorcery/event/tag_set"
+require_relative "goodsorcery/event/request_section"
+require_relative "goodsorcery/event/job_section"
+require_relative "goodsorcery/event/breadcrumb"
+require_relative "goodsorcery/event/builder"
+require_relative "goodsorcery/event_pipeline"
+require_relative "goodsorcery/event_serializer"
+require_relative "goodsorcery/reported_mark"
+require_relative "goodsorcery/ignore_list"
+require_relative "goodsorcery/exception_coercion"
+require_relative "goodsorcery/transport"
+require_relative "goodsorcery/delivery"
+require_relative "goodsorcery/verification"
+require_relative "goodsorcery/test_event"
+require_relative "goodsorcery/client"
+require_relative "goodsorcery/rack/middleware"
+
+# Reports exceptions to a Good Sorcery error tracker. See README.md.
+module GoodSorcery
+  LOCK = Mutex.new
+  @log = Log.new { configuration.logger }
+
+  class << self
+    def configuration
+      @configuration || LOCK.synchronize { @configuration ||= Configuration.new }
+    end
+
+    # Yields the configuration. Rebuilds the delivery when a delivery-related
+    # setting changed, and recompiles filters.
+    def configure
+      before = configuration.delivery_settings
+      yield configuration
+      client.reconfigure(rebuild_delivery: before != configuration.delivery_settings)
+      configuration
+    end
+
+    def client
+      return @client if @client
+
+      config = configuration
+      LOCK.synchronize { @client ||= Client.new(config, log) }
+    end
+
+    # The gem's own logger (prefixed, rate-limited; see GoodSorcery::Log).
+    attr_reader :log
+
+    # Reports an exception (or a message). => event id, or nil. Never raises.
+    def notify(exception_or_message, context: nil, tags: nil, user: nil, level: nil,
+               fingerprint: nil, transaction: nil)
+      options = { context: context, tags: tags, user: user, level: level,
+                  fingerprint: fingerprint, transaction: transaction }.compact
+      client.notify(exception_or_message, **options)
+    rescue StandardError
+      nil
+    end
+
+    def enabled?
+      client.enabled?
+    end
+
+    def scope
+      Scope.current
+    end
+
+    def clear_scope
+      Scope.clear
+    end
+
+    # Runs the block with a fresh scope (one request or job), then restores
+    # the previous one.
+    def with_scope
+      previous = Thread.current[Scope::STORAGE_KEY]
+      Scope.current = Scope.new(unit_of_work: true)
+      yield Scope.current
+    ensure
+      Scope.current = previous
+    end
+
+    def with_context(hash, &)
+      scope.with_context(hash, &)
+    end
+
+    def context(hash)
+      scope.merge_context(hash)
+    end
+
+    def user=(user)
+      scope.user = user
+    end
+
+    def tags(hash)
+      scope.merge_tags(hash)
+    end
+
+    def add_breadcrumb(message, category: "custom", level: "info", data: nil)
+      client.add_breadcrumb(message, category: category, level: level, data: data)
+    end
+
+    def flush(timeout = 5)
+      client.flush(timeout)
+    end
+
+    def shutdown(timeout = nil)
+      @client&.shutdown(timeout || configuration.shutdown_timeout)
+    rescue StandardError
+      nil
+    end
+
+    def verify
+      client.verify
+    end
+
+    def test_events
+      client.test_events
+    end
+
+    def clear_test_events
+      client.clear_test_events
+    end
+
+    # Forgets configuration, client and the current scope. For tests.
+    def reset!
+      shutdown(0.5)
+      LOCK.synchronize do
+        @client = nil
+        @configuration = nil
+      end
+      clear_scope
+    end
+  end
+end
+
+require_relative "goodsorcery/railtie" if defined?(Rails::Railtie)
+
+if defined?(Sidekiq)
+  require_relative "goodsorcery/integrations/sidekiq"
+  GoodSorcery::Integrations::Sidekiq.install
+end

data/plugins.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+# Bundler plugin entry point (`plugin "goodsorcery"` in a Gemfile).
+require_relative "lib/goodsorcery/bundler_plugin"
+
+GoodSorcery::BundlerPlugin.register

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: goodsorcery
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Philip Q Nguyen
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+description: Reports exceptions from Ruby and Rails apps to Good Sorcery. Filters
+  sensitive data in-process and delivers events in batches from a background thread,
+  without blocking the host app.
+email:
+- 5519675+philipqnguyen@users.noreply.github.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- lib/generators/goodsorcery/install/credentials_api_key.rb
+- lib/generators/goodsorcery/install/install_generator.rb
+- lib/generators/goodsorcery/install/templates/initializer.rb
+- lib/goodsorcery.rb
+- lib/goodsorcery/backtrace/code_context.rb
+- lib/goodsorcery/backtrace/line_cache.rb
+- lib/goodsorcery/backtrace/parser.rb
+- lib/goodsorcery/backtrace/paths.rb
+- lib/goodsorcery/bundler_plugin.rb
+- lib/goodsorcery/client.rb
+- lib/goodsorcery/configuration.rb
+- lib/goodsorcery/configuration/validation.rb
+- lib/goodsorcery/defaults.rb
+- lib/goodsorcery/delivery.rb
+- lib/goodsorcery/delivery/active_job_queue.rb
+- lib/goodsorcery/delivery/async.rb
+- lib/goodsorcery/delivery/backoff.rb
+- lib/goodsorcery/delivery/custom.rb
+- lib/goodsorcery/delivery/event_queue.rb
+- lib/goodsorcery/delivery/sync.rb
+- lib/goodsorcery/delivery/test.rb
+- lib/goodsorcery/delivery/worker.rb
+- lib/goodsorcery/delivery_job.rb
+- lib/goodsorcery/env_flag.rb
+- lib/goodsorcery/environment_gate.rb
+- lib/goodsorcery/errors.rb
+- lib/goodsorcery/event/breadcrumb.rb
+- lib/goodsorcery/event/builder.rb
+- lib/goodsorcery/event/cause_chain.rb
+- lib/goodsorcery/event/exception_section.rb
+- lib/goodsorcery/event/fields.rb
+- lib/goodsorcery/event/job_section.rb
+- lib/goodsorcery/event/request_section.rb
+- lib/goodsorcery/event/runtime_context.rb
+- lib/goodsorcery/event/tag_set.rb
+- lib/goodsorcery/event/user_section.rb
+- lib/goodsorcery/event_filter.rb
+- lib/goodsorcery/event_pipeline.rb
+- lib/goodsorcery/event_serializer.rb
+- lib/goodsorcery/exception_coercion.rb
+- lib/goodsorcery/extra_filter_probe.rb
+- lib/goodsorcery/ignore_list.rb
+- lib/goodsorcery/install_check.rb
+- lib/goodsorcery/integrations.rb
+- lib/goodsorcery/integrations/action_cable.rb
+- lib/goodsorcery/integrations/active_job.rb
+- lib/goodsorcery/integrations/active_job/job_details.rb
+- lib/goodsorcery/integrations/rails.rb
+- lib/goodsorcery/integrations/rails/controller_action.rb
+- lib/goodsorcery/integrations/rails/controller_enricher.rb
+- lib/goodsorcery/integrations/rails/error_subscriber.rb
+- lib/goodsorcery/integrations/rails/notification_breadcrumbs.rb
+- lib/goodsorcery/integrations/rails/sql_statement.rb
+- lib/goodsorcery/integrations/rails/user_lookup.rb
+- lib/goodsorcery/integrations/rake.rb
+- lib/goodsorcery/integrations/sidekiq.rb
+- lib/goodsorcery/integrations/sidekiq/job_details.rb
+- lib/goodsorcery/job_arguments.rb
+- lib/goodsorcery/key_filter.rb
+- lib/goodsorcery/log.rb
+- lib/goodsorcery/masked_key.rb
+- lib/goodsorcery/rack/middleware.rb
+- lib/goodsorcery/rack/request_details.rb
+- lib/goodsorcery/rack/request_enricher.rb
+- lib/goodsorcery/rails_filter_parameters.rb
+- lib/goodsorcery/railtie.rb
+- lib/goodsorcery/release_detector.rb
+- lib/goodsorcery/reported_mark.rb
+- lib/goodsorcery/sanitizer.rb
+- lib/goodsorcery/scope.rb
+- lib/goodsorcery/status_report.rb
+- lib/goodsorcery/tasks.rb
+- lib/goodsorcery/test_event.rb
+- lib/goodsorcery/text.rb
+- lib/goodsorcery/transport.rb
+- lib/goodsorcery/transport/batch_chunks.rb
+- lib/goodsorcery/transport/connection.rb
+- lib/goodsorcery/transport/result.rb
+- lib/goodsorcery/verification.rb
+- lib/goodsorcery/version.rb
+- plugins.rb
+homepage: https://app.goodsorcery.com
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+  source_code_uri: https://github.com/creativesorcery/goodsorcery-ruby
+  changelog_uri: https://github.com/creativesorcery/goodsorcery-ruby/blob/main/CHANGELOG.md
+  bug_tracker_uri: https://github.com/creativesorcery/goodsorcery-ruby/issues
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.2'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.16
+specification_version: 4
+summary: Error reporting client for the Good Sorcery error tracker.
+test_files: []