honeybadger 1.16.7 → 2.0.0.beta.1

data/lib/honeybadger/util/request_sanitizer.rb

@@ -0,0 +1,35 @@
+module Honeybadger
+  module Util
+    class RequestSanitizer
+
+      def initialize(sanitizer)
+        @sanitizer = sanitizer
+      end
+
+      def sanitize(request_hash)
+        request_hash.merge({
+          url: sanitize_url(request_hash[:url]),
+          component: request_hash[:component],
+          action: request_hash[:action],
+          params: sanitize_hash(request_hash[:params]),
+          session: sanitize_hash(request_hash[:session]),
+          cgi_data: sanitize_hash(request_hash[:cgi_data])
+        })
+      end
+
+      private
+
+      def sanitize_url(url)
+        if url
+          @sanitizer.filter_url(url)
+        end
+      end
+
+      def sanitize_hash(hash)
+        if hash
+          @sanitizer.filter(@sanitizer.sanitize(hash))
+        end
+      end
+    end
+  end
+end

data/lib/honeybadger/util/sanitizer.rb

@@ -0,0 +1,71 @@
+module Honeybadger
+  module Util
+    class Sanitizer
+      OBJECT_WHITELIST = [Hash, Array, String, Integer, Float, TrueClass, FalseClass, NilClass]
+
+      def initialize(opts = {})
+        @max_depth = opts[:max_depth] || 20
+        @filters = Array(opts[:filters])
+      end
+
+      # Removes non-serializable data and truncates to max depth.
+      def sanitize(data, depth = 0, stack = [])
+        return '[possible infinite recursion halted]' if stack.any?{|item| item == data.object_id }
+
+        if data.respond_to?(:to_hash)
+          return '[max depth reached]' if depth >= max_depth
+          data.to_hash.reduce({}) do |result, (key, value)|
+            result.merge(key => sanitize(value, depth+1, stack + [data.object_id]))
+          end
+        elsif data.respond_to?(:to_ary)
+          return '[max depth reached]' if depth >= max_depth
+          data.to_ary.collect do |value|
+            sanitize(value, depth+1, stack + [data.object_id])
+          end
+        elsif OBJECT_WHITELIST.any? {|c| data.kind_of?(c) }
+          data
+        else
+          data.to_s
+        end
+      end
+
+      def filter(hash)
+        {}.tap do |filtered_hash|
+          hash.each_pair do |key, value|
+            if value.respond_to?(:to_hash)
+              filtered_hash[key] = filter(hash[key])
+            else
+              filtered_hash[key] = filter_key?(key) ? '[FILTERED]' : hash[key]
+            end
+          end
+        end
+      end
+
+      def filter_url(url)
+        filtered_url = url.to_s.dup
+        filtered_url.scan(/(?:^|&|\?)([^=?&]+)=([^&]+)/).each do |m|
+          next unless filter_key?(m[0])
+          filtered_url.gsub!(/#{m[1]}/, '[FILTERED]')
+        end
+
+        filtered_url
+      end
+
+      private
+
+      attr_reader :max_depth, :filters
+
+      def filter_key?(key)
+        return false unless filters
+
+        filters.any? do |filter|
+          if filter.is_a?(Regexp)
+            key.to_s =~ filter
+          else
+            key.to_s.eql?(filter.to_s)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/honeybadger/util/stats.rb

@@ -0,0 +1,31 @@
+module Honeybadger
+  module Util
+    class Stats
+      HAS_MEM = File.exists?("/proc/meminfo")
+      HAS_LOAD = File.exists?("/proc/loadavg")
+
+      class << self
+        def all
+          { :mem => memory, :load => load }
+        end
+
+        # From https://github.com/bloopletech/webstats/blob/master/server/data_providers/mem_info.rb
+        def memory
+          out = {}
+          if HAS_MEM
+            out[:total], out[:free], out[:buffers], out[:cached] = IO.readlines("/proc/meminfo")[0..4].map { |l| l =~ /^.*?\: +(.*?) kB$/; ($1.to_i / 1024.0).to_f }
+            out[:free_total] = out[:free] + out[:buffers] + out[:cached]
+          end
+          out
+        end
+
+        # From https://github.com/bloopletech/webstats/blob/master/server/data_providers/cpu_info.rb
+        def load
+          out = {}
+          out[:one], out[:five], out[:fifteen] = IO.read("/proc/loadavg").split(' ', 4).map(&:to_f) if HAS_LOAD
+          out
+        end
+      end
+    end
+  end
+end

data/lib/honeybadger/version.rb

@@ -0,0 +1,4 @@
+module Honeybadger
+  # Public: The current String Honeybadger version.
+  VERSION = '2.0.0.beta.1'.freeze
+end

data/lib/honeybadger/worker.rb

@@ -0,0 +1,224 @@
+require 'forwardable'
+require 'net/http'
+
+require 'honeybadger/logging'
+
+module Honeybadger
+  class Worker
+    extend Forwardable
+
+    include Honeybadger::Logging::Helper
+
+    autoload :Batch, 'honeybadger/worker/batch'
+    autoload :MetricsCollector, 'honeybadger/worker/metrics_collector'
+    autoload :MeteredQueue, 'honeybadger/worker/metered_queue'
+
+    # Sub-class thread so we have a named thread (useful for debugging in Thread.list).
+    class Thread < ::Thread; end
+
+    attr_reader :backend, :queue, :features, :metrics, :traces, :pid, :mutex, :thread
+
+    def initialize(config)
+      @backend = config.backend
+      @config = config
+      @mutex = Mutex.new
+      prepare
+    end
+
+    def start
+      debug { 'starting worker' }
+
+      @pid = Process.pid
+      @thread = Thread.new { run }
+
+      true
+    end
+
+    def stop(force = false)
+      debug { 'stopping worker' }
+      if thread
+        if force
+          debug { 'killing worker' }
+          Thread.kill(thread)
+        else
+          thread[:should_exit] = true
+          unless thread.eql?(Thread.current)
+            mutex.unlock if mutex.locked?
+            thread.join
+          end
+        end
+      end
+      @thread = nil
+      @pid = nil
+    end
+
+    def fork
+      debug { 'forking worker' }
+
+      stop
+
+      mutex.synchronize { prepare }
+
+      start
+    end
+
+    def notice(notice)
+      debug { sprintf('worker adding notice feature=notices id=%s', notice.id) }
+      push(:notices, notice)
+    end
+
+    def trace(trace)
+      if trace.duration > config[:'traces.threshold']
+        debug { sprintf('worker adding trace duration=%s feature=traces id=%s', trace.duration.round(2), trace.id) }
+        mutex.synchronize do
+          traces.push(trace)
+        end
+      else
+        debug { sprintf('worker discarding trace duration=%s feature=traces id=%s', trace.duration.round(2), trace.id) }
+      end
+    end
+
+    def_delegators :@metrics, :timing, :increment
+
+    private
+
+    attr_reader :config
+
+    def init_queue
+      @queue = {
+        notices: MeteredQueue.new,
+        metrics: MeteredQueue.new,
+        traces: MeteredQueue.new
+      }.freeze
+
+      @features = {
+        notices: true,
+        metrics: true,
+        traces: true
+      }.freeze
+    end
+
+    def init_traces
+      @traces = Batch.new(config, :traces, 20, config[:debug] ? 10 : 60)
+    end
+
+    def init_metrics
+      @metrics = MetricsCollector.new(config, config[:debug] ? 10 : 60)
+    end
+
+    def flush_metrics
+      debug { 'worker flushing metrics feature=metrics' } # TODO: Include count.
+      metrics.chunk(100, &method(:push).to_proc.curry[:metrics])
+      mutex.synchronize { init_metrics }
+    end
+
+    def flush_traces
+      debug { sprintf('worker flushing traces feature=traces count=%d', traces.size) }
+      push(:traces, traces) unless traces.empty?
+      mutex.synchronize { init_traces }
+    end
+
+    def flush_queue
+      mutex.synchronize do
+        queue.each_pair do |feature, queue|
+          while payload = queue.pop!
+            handle_response(feature, notify_backend(feature, payload))
+          end
+        end
+      end
+    end
+
+    def prepare
+      init_queue
+      init_metrics
+      init_traces
+    end
+
+    def push(feature, object)
+      unless features[feature]
+        debug { sprintf('worker dropping feature=%s reason=collector', feature) }
+        return false
+      end
+
+      unless config.features[feature]
+        debug { sprintf('worker dropping feature=%s reason=ping', feature) }
+        return false
+      end
+
+      mutex.synchronize do
+        queue[feature].push(object)
+      end
+
+      true
+    end
+
+    def run
+      begin
+        debug { 'worker started' }
+        work until finish
+      rescue Exception => e
+        error(sprintf('error in worker thread (shutting down) class=%s message=%s location=%s', e.class, e.message.dump, e.backtrace.first.dump))
+        raise e
+      ensure
+        debug { 'stopping worker' }
+      end
+    end
+
+    def work
+      flush_metrics if metrics.flush?
+      flush_traces if traces.flush?
+
+      queue.each_pair do |feature, queue|
+        if payload = queue.pop
+          handle_response(feature, notify_backend(feature, payload))
+        end
+      end
+
+      sleep(0.1)
+    rescue StandardError => e
+      error(sprintf('error in worker thread class=%s message=%s location=%s', e.class, e.message.dump, e.backtrace.first.dump))
+      sleep(1)
+    end
+
+    def finish
+      if Thread.current[:should_exit]
+        debug { 'flushing worker data' }
+
+        flush_metrics
+        flush_traces
+        flush_queue
+
+        true
+      end
+    end
+
+    def notify_backend(feature, payload)
+      debug { sprintf('worker notifying backend feature=%s id=%s', feature, payload.id) }
+      backend.notify(feature, payload)
+    end
+
+    def handle_response(feature, response)
+      debug { sprintf('worker response feature=%s code=%s message=%s', feature, response.code, response.message.to_s.dump) }
+
+      case response.code
+      when 429, 503
+        debug { sprintf('worker applying throttle=1.25 feature=%s code=%s', feature, response.code) }
+        queue[feature].throttle(1.25)
+      when 402
+        warn { sprintf('worker disabling feature=%s code=%s', feature, response.code) }
+        mutex.synchronize { features[feature] = false }
+      when 403
+        error { sprintf('worker shutting down (unauthorized) feature=%s code=%s', feature, response.code) }
+        Honeybadger::Agent.stop(true)
+      when 201
+        if throttle = queue[feature].unthrottle
+          debug { sprintf('worker removing throttle=%s feature=%s code=%s', throttle, feature, response.code) }
+        end
+      when :error
+        # Error logged by backend.
+      else
+        warn { sprintf('worker unknown response feature=%s code=%s', feature, response.code) }
+      end
+    end
+  end
+end

data/lib/honeybadger/worker/batch.rb

@@ -0,0 +1,50 @@
+require 'securerandom'
+
+module Honeybadger
+  class Worker
+    class Batch
+      def initialize(config, name = :data, max = 100, interval = 60, now = now)
+        @id = SecureRandom.uuid
+        @config = config
+        @name = name
+        @max = max
+        @interval = interval
+        @future = now + interval
+        @values = Array.new
+        @mutex = Mutex.new
+      end
+
+      attr_reader :id
+
+      def push(val)
+        mutex.synchronize { values.push(val) }
+      end
+
+      def empty?
+        mutex.synchronize { values.empty? }
+      end
+
+      def size
+        mutex.synchronize { values.size }
+      end
+
+      def flush?
+        size >= max || now >= future
+      end
+
+      def as_json(*args)
+        mutex.synchronize do
+          { name => values.compact.map(&:to_h), :environment => config[:env], :hostname => config[:hostname] }
+        end
+      end
+
+      private
+
+      attr_reader :config, :name, :max, :values, :future, :mutex
+
+      def now
+        Time.now.to_i
+      end
+    end
+  end
+end

data/lib/honeybadger/worker/metered_queue.rb

@@ -0,0 +1,80 @@
+module Honeybadger
+  class Worker
+    # Internal: A thread-safe first-in-first-out queue. Values are pushed onto
+    # the queue and released at a defined interval.
+    class MeteredQueue
+      def initialize(interval = 1, max = 1000, now = now)
+        @interval = interval
+        @max = max
+        @values = Array.new
+        @throttles = Array.new
+        @future = calculate_future(now, interval)
+        @mutex = Mutex.new
+      end
+
+      def size
+        mutex.synchronize { values.size }
+      end
+
+      def push(value)
+        unless values.size == max
+          mutex.synchronize { values.push(value) }
+        end
+      end
+
+      def pop
+        if now.to_i >= future
+          mutex.synchronize do
+            @future = calculate_future
+            values.shift
+          end
+        end
+      end
+
+      def pop!
+        mutex.synchronize { values.shift }
+      end
+
+      # Applies a new throttle to this queue and adjusts the future.
+      #
+      # Returns nothing
+      def throttle(throttle)
+        mutex.synchronize do
+          old_interval = throttled_interval
+          throttles << throttle
+          @future += (throttled_interval - old_interval)
+        end
+      end
+
+      # Removes the last throttle from this queue and adjusts the future.
+      #
+      # Returns Float throttle
+      def unthrottle
+        mutex.synchronize do
+          old_interval = throttled_interval
+          throttles.pop.tap do |throttle|
+            if throttle
+              @future -= (old_interval - throttled_interval)
+            end
+          end
+        end
+      end
+
+      private
+
+      attr_reader :interval, :max, :values, :throttles, :future, :mutex
+
+      def now
+        Time.now
+      end
+
+      def calculate_future(now = now, interval = interval)
+        now.to_i + throttled_interval(interval)
+      end
+
+      def throttled_interval(interval = interval)
+        throttles.reduce(interval) {|a,e| a * e }
+      end
+    end
+  end
+end