celerbrake-ruby 0.1.0

data/lib/celerbrake-ruby/tdigest.rb

@@ -0,0 +1,379 @@
+require 'rbtree'
+
+module Celerbrake
+  # Ruby implementation of Ted Dunning's t-digest data structure.
+  #
+  # This implementation is imported from https://github.com/castle/tdigest with
+  # custom modifications. Huge thanks to Castle for the implementation :beer:
+  #
+  # The difference is that we pack with Big Endian (unlike Native Endian in
+  # Castle's version). Our backend does not permit little endian.
+  #
+  # @see https://github.com/tdunning/t-digest
+  # @see https://github.com/castle/tdigest
+  # @api private
+  # @since v3.2.0
+  #
+  # rubocop:disable Metrics/ClassLength
+  class TDigest
+    VERBOSE_ENCODING = 1
+    SMALL_ENCODING   = 2
+
+    # Centroid represents a number of data points.
+    # @api private
+    # @since v3.2.0
+    class Centroid
+      attr_accessor :mean, :n, :cumn, :mean_cumn
+
+      def initialize(mean, n, cumn, mean_cumn = nil)
+        @mean      = mean
+        @n         = n
+        @cumn      = cumn
+        @mean_cumn = mean_cumn
+      end
+
+      def as_json(_ = nil)
+        { m: mean, n: n }
+      end
+    end
+
+    attr_accessor :centroids
+    attr_reader :size
+
+    def initialize(delta = 0.01, k = 25, cx = 1.1)
+      @delta = delta
+      @k = k
+      @cx = cx
+      @centroids = RBTree.new
+      @size = 0
+      @last_cumulate = 0
+    end
+
+    def +(other)
+      # Uses delta, k and cx from the caller
+      t = self.class.new(@delta, @k, @cx)
+      data = centroids.values + other.centroids.values
+      t.push_centroid(data.delete_at(rand(data.length))) while data.any?
+      t
+    end
+
+    def as_bytes
+      # compression as defined by Java implementation
+      size = @centroids.size
+      output = [VERBOSE_ENCODING, compression, size]
+      output += @centroids.each_value.map(&:mean)
+      output += @centroids.each_value.map(&:n)
+      output.pack("NGNG#{size}N#{size}")
+    end
+
+    # rubocop:disable Metrics/AbcSize
+    def as_small_bytes
+      size = @centroids.size
+      output = [self.class::SMALL_ENCODING, compression, size]
+      x = 0
+      # delta encoding allows saving 4-bytes floats
+      mean_arr = @centroids.each_value.map do |c|
+        val = c.mean - x
+        x = c.mean
+        val
+      end
+      output += mean_arr
+      # Variable length encoding of numbers
+      c_arr = @centroids.each_value.with_object([]) do |c, arr|
+        k = 0
+        n = c.n
+        while n < 0 || n > 0x7f
+          b = 0x80 | (0x7f & n)
+          arr << b
+          n = n >> 7
+          k += 1
+          raise 'Unreasonable large number' if k > 6
+        end
+        arr << n
+      end
+      output += c_arr
+      output.pack("NGNg#{size}C#{size}")
+    end
+    # rubocop:enable Metrics/AbcSize
+
+    def as_json(_ = nil)
+      @centroids.each_value.map(&:as_json)
+    end
+
+    def bound_mean(x)
+      upper = @centroids.upper_bound(x)
+      lower = @centroids.lower_bound(x)
+      [lower[1], upper[1]]
+    end
+
+    def bound_mean_cumn(cumn)
+      last_c = nil
+      bounds = []
+      @centroids.each_value do |v|
+        if v.mean_cumn == cumn
+          bounds << v
+          break
+        elsif v.mean_cumn > cumn
+          bounds << last_c
+          bounds << v
+          break
+        else
+          last_c = v
+        end
+      end
+      # If still no results, pick lagging value if any
+      bounds << last_c if bounds.empty? && !last_c.nil?
+
+      bounds
+    end
+
+    def compress!
+      points = to_a
+      reset!
+      push_centroid(points.shuffle)
+      _cumulate(exact: true, force: true)
+      nil
+    end
+
+    def compression
+      1 / @delta
+    end
+
+    def find_nearest(x)
+      return if size == 0
+
+      upper_key, upper = @centroids.upper_bound(x)
+      lower_key, lower = @centroids.lower_bound(x)
+      return lower unless upper_key
+      return upper unless lower_key
+
+      if (lower_key - x).abs < (upper_key - x).abs
+        lower
+      else
+        upper
+      end
+    end
+
+    def merge!(other)
+      push_centroid(other.centroids.values.shuffle)
+      self
+    end
+
+    # rubocop:disable Metrics/PerceivedComplexity, Metrics/AbcSize
+    # rubocop:disable Metrics/CyclomaticComplexity
+    def p_rank(x)
+      is_array = x.is_a? Array
+      x = [x] unless is_array
+
+      min = @centroids.first
+      max = @centroids.last
+
+      x.map! do |item|
+        if size == 0
+          nil
+        elsif item < min[1].mean
+          0.0
+        elsif item > max[1].mean
+          1.0
+        else
+          _cumulate(exact: true)
+          bound = bound_mean(item)
+          lower, upper = bound
+          mean_cumn = lower.mean_cumn
+          if lower != upper
+            mean_cumn += (item - lower.mean) * (upper.mean_cumn - lower.mean_cumn) \
+              / (upper.mean - lower.mean)
+          end
+          mean_cumn / @size
+        end
+      end
+      is_array ? x : x.first
+    end
+    # rubocop:enable Metrics/PerceivedComplexity, Metrics/AbcSize
+    # rubocop:enable Metrics/CyclomaticComplexity
+
+    # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+    # rubocop:disable Metrics/AbcSize
+    def percentile(p)
+      is_array = p.is_a? Array
+      p = [p] unless is_array
+      p.map! do |item|
+        unless (0..1).cover?(item)
+          raise ArgumentError, "p should be in [0,1], got #{item}"
+        end
+
+        if size == 0
+          nil
+        else
+          _cumulate(exact: true)
+          h = @size * item
+          lower, upper = bound_mean_cumn(h)
+          if lower.nil? && upper.nil?
+            nil
+          elsif upper == lower || lower.nil? || upper.nil?
+            (lower || upper).mean
+          elsif h == lower.mean_cumn
+            lower.mean
+          else
+            upper.mean
+          end
+        end
+      end
+      is_array ? p : p.first
+    end
+    # rubocop:enable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+    # rubocop:enable Metrics/AbcSize
+
+    def push(x, n = 1)
+      x = [x] unless x.is_a? Array
+      x.each { |value| _digest(value, n) }
+    end
+
+    def push_centroid(c)
+      c = [c] unless c.is_a? Array
+      c.each { |centroid| _digest(centroid.mean, centroid.n) }
+    end
+
+    def reset!
+      @centroids.clear
+      @size = 0
+      @last_cumulate = 0
+    end
+
+    def to_a
+      @centroids.each_value.to_a
+    end
+
+    # rubocop:disable Metrics/PerceivedComplexity, Metrics/MethodLength
+    # rubocop:disable Metrics/CyclomaticComplexity, Metrics/AbcSize
+    def self.from_bytes(bytes)
+      format, compression, size = bytes.unpack('NGN')
+      tdigest = new(1 / compression)
+
+      start_idx = 16 # after header
+      case format
+      when VERBOSE_ENCODING
+        array = bytes[start_idx..-1].unpack("G#{size}N#{size}")
+        means, counts = array.each_slice(size).to_a if array.any?
+      when SMALL_ENCODING
+        means = bytes[start_idx..(start_idx + (4 * size))].unpack("g#{size}")
+        # Decode delta encoding of means
+        x = 0
+        means.map! do |m|
+          m += x
+          x = m
+          m
+        end
+        counts_bytes = bytes[(start_idx + (4 * size))..-1].unpack('C*')
+        counts = []
+        # Decode variable length integer bytes
+        size.times do
+          v = counts_bytes.shift
+          z = 0x7f & v
+          shift = 7
+          while (v & 0x80) != 0
+            raise 'Shift too large in decode' if shift > 28
+
+            v = counts_bytes.shift || 0
+            z += (v & 0x7f) << shift
+            shift += 7
+          end
+          counts << z
+        end
+        # This shouldn't happen
+        raise 'Mismatch' unless counts.size == means.size
+      else
+        raise 'Unknown compression format'
+      end
+
+      means.zip(counts).each { |val| tdigest.push(val[0], val[1]) } if means && counts
+
+      tdigest
+    end
+    # rubocop:enable Metrics/PerceivedComplexity, Metrics/MethodLength
+    # rubocop:enable Metrics/CyclomaticComplexity, Metrics/AbcSize
+
+    def self.from_json(array)
+      tdigest = new
+      # Handle both string and symbol keys
+      array.each { |a| tdigest.push(a['m'] || a[:m], a['n'] || a[:n]) }
+      tdigest
+    end
+
+    private
+
+    def _add_weight(centroid, x, n)
+      unless x == centroid.mean
+        centroid.mean += n * (x - centroid.mean) / (centroid.n + n)
+      end
+
+      _cumulate(exact: false, force: true) if centroid.mean_cumn.nil?
+
+      centroid.cumn += n
+      centroid.mean_cumn += n / 2.0
+      centroid.n += n
+    end
+
+    # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+    def _cumulate(exact: false, force: false)
+      unless force
+        factor = if @last_cumulate == 0
+                   Float::INFINITY
+                 else
+                   (@size.to_f / @last_cumulate)
+                 end
+        return if @size == @last_cumulate || (!exact && @cx && @cx > factor)
+      end
+
+      cumn = 0
+      @centroids.each_value do |c|
+        c.mean_cumn = cumn + (c.n / 2.0)
+        cumn = c.cumn = cumn + c.n
+      end
+      @size = @last_cumulate = cumn
+      nil
+    end
+    # rubocop:enable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+
+    # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+    # rubocop:disable Metrics/AbcSize
+    def _digest(x, n)
+      # Use 'first' and 'last' instead of min/max because of performance reasons
+      # This works because RBTree is sorted
+      min = min.last if (min = @centroids.first)
+      max = max.last if (max = @centroids.last)
+      nearest = find_nearest(x)
+
+      @size += n
+
+      if nearest && nearest.mean == x
+        _add_weight(nearest, x, n)
+      elsif nearest == min
+        @centroids[x] = Centroid.new(x, n, 0)
+      elsif nearest == max
+        @centroids[x] = Centroid.new(x, n, @size)
+      else
+        p = nearest.mean_cumn.to_f / @size
+        max_n = (4 * @size * @delta * p * (1 - p)).floor
+        if max_n - nearest.n >= n
+          _add_weight(nearest, x, n)
+        else
+          @centroids[x] = Centroid.new(x, n, nearest.cumn)
+        end
+      end
+
+      _cumulate(exact: false)
+
+      # If the number of centroids has grown to a very large size,
+      # it may be due to values being inserted in sorted order.
+      # We combat that by replaying the centroids in random order,
+      # which is what compress! does
+      compress! if @centroids.size > (@k / @delta)
+
+      nil
+    end
+    # rubocop:enable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity,
+    # rubocop:enable Metrics/AbcSize
+  end
+  # rubocop:enable Metrics/ClassLength
+end

data/lib/celerbrake-ruby/thread_pool.rb

@@ -0,0 +1,139 @@
+module Celerbrake
+  # ThreadPool implements a simple thread pool that can configure the number of
+  # worker threads and the size of the queue to process.
+  #
+  # @example
+  #   # Initialize a new thread pool with 5 workers and a queue size of 100. Set
+  #   # the block to be run concurrently.
+  #   thread_pool = ThreadPool.new(
+  #     name: 'performance-notifier',
+  #     worker_size: 5,
+  #     queue_size: 100,
+  #     block: proc { |message| print "ECHO: #{message}..."}
+  #   )
+  #
+  #   # Send work.
+  #   10.times { |i| thread_pool << i }
+  #   #=> ECHO: 0...ECHO: 1...ECHO: 2...
+  #
+  # @api private
+  # @since v4.6.1
+  class ThreadPool
+    include Loggable
+
+    # @return [ThreadGroup] the list of workers
+    # @note This is exposed for eaiser unit testing
+    attr_reader :workers
+
+    def initialize(worker_size:, queue_size:, block:, name: nil)
+      @name = name
+      @worker_size = worker_size
+      @queue_size = queue_size
+      @block = block
+
+      @queue = SizedQueue.new(queue_size)
+      @workers = ThreadGroup.new
+      @mutex = Mutex.new
+      @pid = nil
+      @closed = false
+
+      has_workers?
+    end
+
+    # Adds a new message to the thread pool. Rejects messages if the queue is at
+    # its capacity.
+    #
+    # @param [Object] message The message that gets passed to the block
+    # @return [Boolean] true if the message was successfully sent to the pool,
+    #   false if the queue is full
+    def <<(message)
+      if backlog >= @queue_size
+        logger.info do
+          "#{LOG_LABEL} ThreadPool has reached its capacity of " \
+          "#{@queue_size} and the following message will not be " \
+          "processed: #{message.inspect}"
+        end
+        return false
+      end
+
+      @queue << message
+      true
+    end
+
+    # @return [Integer] how big the queue is at the moment
+    def backlog
+      @queue.size
+    end
+
+    # Checks if a thread pool has any workers. A thread pool doesn't have any
+    # workers only in two cases: when it was closed or when all workers
+    # crashed. An *active* thread pool doesn't have any workers only when
+    # something went wrong.
+    #
+    # Workers are expected to crash when you +fork+ the process the workers are
+    # living in. In this case we detect a +fork+ and try to revive them here.
+    #
+    # Another possible scenario that crashes workers is when you close the
+    # instance on +at_exit+, but some other +at_exit+ hook prevents the process
+    # from exiting.
+    #
+    # @return [Boolean] true if an instance wasn't closed, but has no workers
+    # @see https://goo.gl/oydz8h Example of at_exit that prevents exit
+    def has_workers?
+      @mutex.synchronize do
+        return false if @closed
+
+        if @pid != Process.pid && @workers.list.empty?
+          @pid = Process.pid
+          @workers = ThreadGroup.new
+          spawn_workers
+        end
+
+        !@closed && @workers.list.any?
+      end
+    end
+
+    # Closes the thread pool making it a no-op (it shut downs all worker
+    # threads). Before closing, waits on all unprocessed tasks to be processed.
+    #
+    # @return [void]
+    # @raise [Celerbrake::Error] when invoked more than one time
+    def close
+      threads = @mutex.synchronize do
+        raise Celerbrake::Error, 'this thread pool is closed already' if @closed
+
+        unless @queue.empty?
+          msg = "#{LOG_LABEL} waiting to process #{@queue.size} task(s)..."
+          logger.debug("#{msg} (Ctrl-C to abort)")
+        end
+
+        @worker_size.times { @queue << :stop }
+        @closed = true
+        @workers.list.dup
+      end
+
+      threads.each(&:join)
+      logger.debug("#{LOG_LABEL} #{@name} thread pool closed")
+    end
+
+    def closed?
+      @closed
+    end
+
+    def spawn_workers
+      @worker_size.times { @workers.add(spawn_worker) }
+    end
+
+    private
+
+    def spawn_worker
+      Thread.new do
+        while (message = @queue.pop)
+          break if message == :stop
+
+          @block.call(message)
+        end
+      end
+    end
+  end
+end

data/lib/celerbrake-ruby/time_truncate.rb

@@ -0,0 +1,17 @@
+module Celerbrake
+  # TimeTruncate contains methods for truncating time.
+  #
+  # @api private
+  # @since v3.2.0
+  module TimeTruncate
+    # Truncate +time+ to floor minute and turn it into an RFC3339 timestamp.
+    #
+    # @param [Time, Integer, Float] time
+    # @return [String]
+    def self.utc_truncate_minutes(time)
+      tm = Time.at(time).getutc
+
+      Time.utc(tm.year, tm.month, tm.day, tm.hour, tm.min).to_datetime.rfc3339
+    end
+  end
+end

data/lib/celerbrake-ruby/timed_trace.rb

@@ -0,0 +1,56 @@
+module Celerbrake
+  # TimedTrace represents a chunk of code performance of which was measured and
+  # stored under a label. The chunk is called a "span".
+  #
+  # @example
+  #   timed_trace = TimedTrace.new
+  #   timed_trace.span('http request') do
+  #     http.get('example.com')
+  #   end
+  #   timed_trace.spans #=> { 'http request' => 0.123 }
+  #
+  # @api public
+  # @since v4.3.0
+  class TimedTrace
+    # @param [String] label
+    # @return [Celerbrake::TimedTrace]
+    def self.span(label, &block)
+      new.tap { |timed_trace| timed_trace.span(label, &block) }
+    end
+
+    def initialize
+      @spans = {}
+    end
+
+    # @param [String] label
+    # @return [Boolean]
+    def span(label)
+      start_span(label)
+      yield
+      stop_span(label)
+    end
+
+    # @param [String] label
+    # @return [Boolean]
+    def start_span(label)
+      return false if @spans.key?(label)
+
+      @spans[label] = Celerbrake::Benchmark.new
+      true
+    end
+
+    # @param [String] label
+    # @return [Boolean]
+    def stop_span(label)
+      return false unless @spans.key?(label)
+
+      @spans[label].stop
+      true
+    end
+
+    # @return [Hash<String=>Float>]
+    def spans
+      @spans.transform_values(&:duration)
+    end
+  end
+end

data/lib/celerbrake-ruby/truncator.rb

@@ -0,0 +1,121 @@
+module Celerbrake
+  # This class is responsible for truncation of too big objects. Mainly, you
+  # should use it for simple objects such as strings, hashes, & arrays.
+  #
+  # @api private
+  # @since v1.0.0
+  class Truncator
+    # @return [Hash] the options for +String#encode+
+    ENCODING_OPTIONS = { invalid: :replace, undef: :replace }.freeze
+
+    # @return [String] the temporary encoding to be used when fixing invalid
+    #   strings with +ENCODING_OPTIONS+
+    TEMP_ENCODING = 'utf-16'.freeze
+
+    # @return [Array<Encoding>] encodings that are eligible for fixing invalid
+    #   characters
+    SUPPORTED_ENCODINGS = [Encoding::UTF_8, Encoding::ASCII].freeze
+
+    # @return [String] what to append when something is a circular reference
+    CIRCULAR = '[Circular]'.freeze
+
+    # @return [String] what to append when something is truncated
+    TRUNCATED = '[Truncated]'.freeze
+
+    # @return [Array<Class>] The types that can contain references to itself
+    CIRCULAR_TYPES = [Array, Hash, Set].freeze
+
+    # @param [Integer] max_size maximum size of hashes, arrays and strings
+    def initialize(max_size)
+      @max_size = max_size
+    end
+
+    # Performs deep truncation of arrays, hashes, sets & strings. Uses a
+    # placeholder for recursive objects (`[Circular]`).
+    #
+    # @param [Object] object The object to truncate
+    # @param [Set] seen The cache that helps to detect recursion
+    # @return [Object] truncated object
+    def truncate(object, seen = Set.new)
+      if seen.include?(object.object_id)
+        return CIRCULAR if CIRCULAR_TYPES.any? { |t| object.is_a?(t) }
+
+        return object
+      end
+      truncate_object(object, seen << object.object_id)
+    end
+
+    # Reduces maximum allowed size of hashes, arrays, sets & strings by half.
+    # @return [Integer] current +max_size+ value
+    def reduce_max_size
+      @max_size /= 2
+    end
+
+    private
+
+    def truncate_object(object, seen)
+      case object
+      when Hash then truncate_hash(object, seen)
+      when Array then truncate_array(object, seen)
+      when Set then truncate_set(object, seen)
+      when String then truncate_string(object)
+      when Numeric, TrueClass, FalseClass, Symbol, NilClass then object
+      else
+        truncate_string(stringify_object(object))
+      end
+    end
+
+    def truncate_string(str)
+      fixed_str = replace_invalid_characters(str)
+      return fixed_str if fixed_str.length <= @max_size
+
+      (fixed_str.slice(0, @max_size) + TRUNCATED).freeze
+    end
+
+    def stringify_object(object)
+      object.to_json
+    rescue *Notice::JSON_EXCEPTIONS
+      object.to_s
+    end
+
+    def truncate_hash(hash, seen)
+      truncated_hash = {}
+      hash.each_with_index do |(key, val), idx|
+        break if idx + 1 > @max_size
+
+        truncated_hash[key] = truncate(val, seen)
+      end
+
+      truncated_hash.freeze
+    end
+
+    def truncate_array(array, seen)
+      array.slice(0, @max_size).map! { |elem| truncate(elem, seen) }.freeze
+    end
+
+    def truncate_set(set, seen)
+      truncated_set = Set.new
+
+      set.each do |elem|
+        truncated_set << truncate(elem, seen)
+        break if truncated_set.size >= @max_size
+      end
+
+      truncated_set.freeze
+    end
+
+    # Replaces invalid characters in a string with arbitrary encoding.
+    #
+    # @param [String] str The string to replace characters
+    # @return [String] a UTF-8 encoded string
+    # @see https://github.com/flori/json/commit/3e158410e81f94dbbc3da6b7b35f4f64983aa4e3
+    def replace_invalid_characters(str)
+      utf8_string = SUPPORTED_ENCODINGS.include?(str.encoding)
+      return str if utf8_string && str.valid_encoding?
+
+      temp_str = str.dup
+      temp_str.encode!(TEMP_ENCODING, **ENCODING_OPTIONS) if utf8_string
+      temp_str.encode!('utf-8', **ENCODING_OPTIONS)
+    end
+  end
+end