inst_statsd 2.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f5fcae338209105c5074036e543c03f719f4b3ae
+  data.tar.gz: dbb3c4e3de0a2138737637b86b27ae9764fa7c27
+SHA512:
+  metadata.gz: 00c31868c01435faa215bccbfaca3675fa86c0f7b7e4a356b57c7c04cdfcf83b20f4a33d31c58c0a46d94731f9c3bf01d1ea0d05f0c6204bfe15cd7daf26bdc6
+  data.tar.gz: 91bec914d1e70b5b497b35e675c9d947f0d8924fce7a20d4b8e9c7d4366becd43fa138f61df7ddc7820ef16e77afdb4764b6d9adaa5f9191cb0f4409d2b982f2

data/lib/inst_statsd.rb

@@ -0,0 +1,65 @@
+require 'statsd'
+
+module InstStatsd
+  VALID_SETTINGS = %i[host port namespace append_hostname mask negative_mask batch_size batch_byte_size].freeze
+
+  class ConfigurationError < StandardError; end
+
+  require 'inst_statsd/statsd'
+  require 'inst_statsd/block_stat'
+  require 'inst_statsd/block_tracking'
+  require 'inst_statsd/request_stat'
+  require 'inst_statsd/counter'
+  require 'inst_statsd/sql_tracker'
+  require 'inst_statsd/default_tracking'
+  require 'inst_statsd/request_logger'
+  require 'inst_statsd/request_tracking'
+  require 'inst_statsd/null_logger'
+
+  class << self
+    def settings
+      @settings ||= env_settings
+    end
+
+    def settings=(value)
+      @settings = validate_settings(value)
+    end
+
+    def validate_settings(value)
+      return nil if value.nil?
+
+      validated = {}
+      value.each do |k, v|
+        unless VALID_SETTINGS.include?(k.to_sym)
+          raise InstStatsd::ConfigurationError, "Invalid key: #{k}"
+        end
+        v = Regexp.new(v) if %i[mask negative_mask].include?(k.to_sym) && v.is_a?(String)
+        validated[k.to_sym] = v
+      end
+
+      env_settings.merge(validated)
+    end
+
+    def env_settings(env = ENV)
+      config = {
+        host: env.fetch('INST_STATSD_HOST', nil),
+        port: env.fetch('INST_STATSD_PORT', nil),
+        namespace: env.fetch('INST_STATSD_NAMESPACE', nil),
+        append_hostname: env.fetch('INST_STATSD_APPEND_HOSTNAME', nil)
+      }
+      config.delete_if { |_k, v| v.nil? }
+      convert_bool(config, :append_hostname)
+      config[:host] ? config : {}
+    end
+
+    def convert_bool(hash, key)
+      value = hash[key]
+      return if value.nil?
+      unless ['true', 'True', 'false', 'False', true, false].include?(value)
+        message = "#{key} must be a boolean, or the string representation of a boolean, got: #{value}"
+        raise InstStatsd::ConfigurationError, message
+      end
+      hash[key] = ['true', 'True', true].include?(value)
+    end
+  end
+end

data/lib/inst_statsd/block_stat.rb

@@ -0,0 +1,37 @@
+module InstStatsd
+  class BlockStat
+
+    attr_accessor :stats
+    attr_accessor :common_key
+
+    def initialize(common_key, statsd=InstStatsd::Statsd)
+      self.common_key = common_key
+      @statsd = statsd
+      @stats = {}
+    end
+
+    def subtract_exclusives(stats)
+      @exclusives ||= {}
+      stats.each do |(key, value)|
+        @exclusives[key] ||= 0.0
+        @exclusives[key] += value
+      end
+    end
+
+    def exclusive_stats
+      return nil unless @exclusives
+      stats.map { |key, value| [key, value - (@exclusives[key] || 0.0)] }.to_h
+    end
+
+    def report
+      if common_key
+        stats.each do |(key, value)|
+          @statsd.timing("#{common_key}.#{key}", value)
+        end
+        exclusive_stats&.each do |(key, value)|
+          @statsd.timing("#{common_key}.exclusive.#{key}", value)
+        end
+      end
+    end
+  end
+end

data/lib/inst_statsd/block_tracking.rb

@@ -0,0 +1,62 @@
+require 'benchmark'
+
+module InstStatsd
+  class BlockTracking
+    class << self
+      attr_accessor :logger
+
+      [:mask, :negative_mask].each do |method|
+        class_eval <<-RUBY, __FILE__, __LINE__ + 1
+          def #{method}
+            InstStatsd.settings[:#{method}]
+          end
+
+          def #{method}=(value)
+            InstStatsd.settings[:#{method}] = value
+          end
+        RUBY
+      end
+
+      def track(key, category: nil, statsd: InstStatsd::Statsd, only: nil)
+        return yield if mask && mask !~ key
+        return yield if negative_mask && negative_mask =~ key
+
+        cookies = if only
+                    Array(only).map { |name| [name, Counter.counters[name].start] }
+                  else
+                    Counter.counters.map { |(name, counter)| [name, counter.start] }
+                  end
+        block_stat = InstStatsd::BlockStat.new(key, statsd)
+        stack(category).push(block_stat) if category
+
+        result = nil
+        elapsed = Benchmark.realtime do
+          result = yield
+        end
+        # to be consistent with ActionPack, measure in milliseconds
+        elapsed *= 1000
+
+        block_stat.stats = cookies.map { |(name, cookie)| [name, Counter.counters[name].finalize_count(cookie)] }.to_h
+        block_stat.stats['total'] = elapsed
+        # we need to make sure to report exclusive timings, even if nobody called us re-entrantly
+        block_stat.subtract_exclusives({}) if category
+        block_stat.report
+        logger.log(block_stat, "STATSD #{key}") if logger
+        # -1 is ourselves; we want to subtract from the block above us
+        stack(category)[-2].subtract_exclusives(block_stat.stats) if category && stack(category)[-2]
+
+        result
+      ensure
+        stack(category).pop if category && stack(category).last == block_stat
+      end
+
+      private
+
+      def stack(category)
+        Thread.current[:stats_block_stack] ||= {}
+        Thread.current[:stats_block_stack][category] ||= []
+      end
+    end
+
+  end
+end

data/lib/inst_statsd/counter.rb

@@ -0,0 +1,44 @@
+module InstStatsd
+  class Counter
+    class << self
+      def counters
+        @counters ||= {}
+      end
+
+      def register(counter)
+        counters[counter.key] = counter
+      end
+    end
+
+    attr_reader :key
+    attr_reader :blocked_names
+
+    def initialize(key, blocked_names=[])
+      @blocked_names = blocked_names
+      @key = key
+      @tls_key = "statsd.#{key}"
+      self.class.register(self)
+    end
+
+    def start
+      Thread.current[@tls_key] ||= 0
+    end
+
+    def track(name)
+      Thread.current[@tls_key] += 1 if Thread.current[@tls_key] && accepted_name?(name)
+    end
+
+    def finalize_count(cookie)
+      Thread.current[@tls_key] - cookie
+    end
+
+    def count
+      Thread.current[@tls_key]
+    end
+
+    def accepted_name?(name)
+      !blocked_names.include?(name)
+    end
+
+  end
+end

data/lib/inst_statsd/default_tracking.rb

@@ -0,0 +1,42 @@
+require "active_support"
+
+module InstStatsd
+  class DefaultTracking
+    def self.track_sql
+      return if @sql_tracker
+      @sql_tracker = InstStatsd::SqlTracker.new(blocked_names: ['SCHEMA'])
+      ActiveSupport::Notifications.subscribe(/sql\.active_record/) {|*args| update_sql_count(*args)}
+    end
+
+    def self.track_active_record
+      return if @ar_counter
+      require 'aroi'
+
+      ::Aroi::Instrumentation.instrument_creation!
+      @ar_counter = InstStatsd::Counter.new('active_record')
+      ActiveSupport::Notifications.subscribe(/instance\.active_record/) {|*args| update_active_record_count(*args)}
+    end
+
+    def self.track_cache
+      return if @cache_read_counter
+
+      @cache_read_counter = InstStatsd::Counter.new('cache.read')
+      ActiveSupport::Notifications.subscribe(/cache_read\.active_support/) {|*args| update_cache_read_count(*args)}
+    end
+
+    private
+
+    def self.update_sql_count(_name, _start, _finish, _id, payload)
+      @sql_tracker.track payload.fetch(:name), payload.fetch(:sql)
+    end
+
+    def self.update_active_record_count(_name, _start, _finish, _id, payload)
+      @ar_counter.track payload.fetch(:name, '')
+    end
+
+    def self.update_cache_read_count(_name, _start, _finish, _id, _payload)
+      @cache_read_counter.track "read"
+    end
+
+  end
+end

data/lib/inst_statsd/null_logger.rb

@@ -0,0 +1,11 @@
+require 'logger'
+
+module InstStatsd
+  class NullLogger < Logger
+    def initialize(*args)
+    end
+
+    def add(*args, &block)
+    end
+  end
+end

data/lib/inst_statsd/request_logger.rb

@@ -0,0 +1,25 @@
+module InstStatsd
+  class RequestLogger
+
+    def initialize(logger)
+      @logger = logger || InstStatsd::NullLogger.new
+    end
+
+    def log(request_stat, header=nil)
+      @logger.info(build_log_message(request_stat, header))
+    end
+
+    def build_log_message(request_stat, header=nil)
+      header ||= "STATSD"
+      message = "[#{header}]"
+      request_stat.stats.each do |(name, value)|
+        message += " (#{name.to_s.gsub('.', '_')}: #{"%.2f" % value})"
+      end
+      request_stat.exclusive_stats&.each do |(name, value)|
+        message += " (exclusive_#{name.to_s.gsub('.', '_')}: #{"%.2f" % value})"
+      end
+      message
+    end
+
+  end
+end

data/lib/inst_statsd/request_stat.rb

@@ -0,0 +1,49 @@
+module InstStatsd
+  class RequestStat < BlockStat
+    def initialize(name, start, finish, id, payload, statsd=InstStatsd::Statsd)
+      super(nil, statsd)
+      @name = name
+      @start = start
+      @finish = finish
+      @id = id
+      @payload = payload
+    end
+
+    def common_key
+      common_key = super
+      return common_key if common_key
+      self.common_key = "request.#{controller}.#{action}" if controller && action
+    end
+
+    def report
+      stats['total'] = total
+      stats['view'] = view_runtime if view_runtime
+      stats['db'] = db_runtime if db_runtime
+      super
+    end
+
+    def db_runtime
+      @payload.fetch(:db_runtime, nil)
+    end
+
+    def view_runtime
+      @payload.fetch(:view_runtime, nil)
+    end
+
+    def controller
+      @payload.fetch(:params, {})['controller']
+    end
+
+    def action
+      @payload.fetch(:params, {})['action']
+    end
+
+    def total
+      if (!@finish || !@start)
+        return 0
+      end
+      (@finish - @start) * 1000
+    end
+
+  end
+end

data/lib/inst_statsd/request_tracking.rb

@@ -0,0 +1,27 @@
+module InstStatsd
+  class RequestTracking
+
+    def self.enable(logger: nil)
+      @logger = RequestLogger.new(logger)
+      track_timing
+    end
+
+    private
+
+    def self.track_timing
+      ActiveSupport::Notifications.subscribe(/start_processing\.action_controller/, &method(:start_processing))
+      ActiveSupport::Notifications.subscribe(/process_action\.action_controller/, &method(:finalize_processing))
+    end
+
+    def self.start_processing(*_args)
+      @cookies = Counter.counters.map { |(name, counter)| [name, counter.start] }
+    end
+
+    def self.finalize_processing *args
+      request_stat = InstStatsd::RequestStat.new(*args)
+      request_stat.stats = @cookies.map { |(name, cookie)| [name, Counter.counters[name].finalize_count(cookie)] }.to_h
+      request_stat.report
+      @logger.log(request_stat)
+    end
+  end
+end

data/lib/inst_statsd/sql_tracker.rb

@@ -0,0 +1,50 @@
+module InstStatsd
+  class SqlTracker
+
+    attr_reader :blocked_names, :read_counts, :write_counts, :cache_counts
+
+    def initialize(opts=nil)
+      opts ||= {}
+      @blocked_names = opts.fetch(:blocked_names, [])
+      @read_counts = opts.fetch(:read_counter, InstStatsd::Counter.new('sql.read'))
+      @write_counts = opts.fetch(:write_counter, InstStatsd::Counter.new('sql.write'))
+      @cache_counts = opts.fetch(:cache_counter, InstStatsd::Counter.new('sql.cache'))
+    end
+
+    def start
+      [read_counts, write_counts, cache_counts].map(&:start)
+    end
+
+    def track name, sql
+      return unless sql && accepted_name?(name)
+
+      if name.match(/CACHE/)
+        cache_counts.track name
+      elsif truncate(sql).match(/SELECT/) || name.match(/LOAD/)
+        read_counts.track(sql)
+      else
+        write_counts.track(sql)
+      end
+    end
+
+    def finalize_counts(cookies)
+      [
+        read_counts.finalize_count(cookies[0]),
+        write_counts.finalize_count(cookies[1]),
+        cache_counts.finalize_count(cookies[2])
+      ]
+    end
+
+    private
+
+    def accepted_name?(name)
+      !!(name && !blocked_names.include?(name))
+    end
+
+    def truncate(sql, length=15)
+      sql ||= ''
+      sql.strip[0..length]
+    end
+
+  end
+end

data/lib/inst_statsd/statsd.rb

@@ -0,0 +1,102 @@
+# Proxy class to communicate messages to statsd
+# Available statsd messages are described in:
+#   https://github.com/etsy/statsd/blob/master/README.md
+#   https://github.com/reinh/statsd/blob/master/lib/statsd.rb
+#
+# So for instance:
+#   ms = Benchmark.ms { ..code.. }
+#   InstStatsd::Statsd.timing("my_stat", ms)
+#
+# Configured in config/statsd.yml, see config/statsd.yml.example
+# At least a host needs to be defined for the environment, all other config is optional
+#
+# If a namespace is defined in statsd.yml, it'll be prepended to the stat name.
+# The hostname of the server will be appended to the stat name, unless `append_hostname: false` is specified in the config.
+# So if the namespace is "canvas" and the hostname is "app01", the final stat name of "my_stat" would be "stats.canvas.my_stat.app01"
+# (assuming the default statsd/graphite configuration)
+#
+# If statsd isn't configured and enabled, then calls to InstStatsd::Statsd.* will do nothing and return nil
+
+module InstStatsd
+  module Statsd
+    # replace "." in key names with another character to avoid creating spurious sub-folders in graphite
+    def self.escape(str, replacement = '_')
+      str.respond_to?(:gsub) ? str.gsub('.', replacement) : str
+    end
+
+    def self.hostname
+      @hostname ||= Socket.gethostname.split('.').first
+    end
+
+    %w[increment decrement count gauge timing].each do |method|
+      class_eval <<-RUBY, __FILE__, __LINE__ + 1
+      def self.#{method}(stat, *args)
+        if self.instance
+          if Array === stat
+            stat.each do |st|
+              self.#{method}(st, *args)
+            end
+            return
+          end
+
+          if self.append_hostname?
+            stat_name = "\#{stat}.\#{hostname}"
+          else
+            stat_name = stat.to_s
+          end
+          self.instance.#{method}(stat_name, *args)
+        else
+          nil
+        end
+      end
+      RUBY
+    end
+
+    def self.time(stat, sample_rate = 1)
+      start = Time.now
+      result = yield
+      timing(stat, ((Time.now - start) * 1000).round, sample_rate)
+      result
+    end
+
+    def self.batch
+      return yield unless (old_instance = instance)
+      old_instance.batch do |batch|
+        Thread.current[:inst_statsd] = batch
+        yield
+      end
+    ensure
+      Thread.current[:inst_statsd] = old_instance
+    end
+
+    def self.instance
+      thread_statsd = Thread.current[:inst_statsd]
+      return thread_statsd if thread_statsd
+
+      unless defined?(@statsd)
+        statsd_settings = InstStatsd.settings
+
+        if statsd_settings && statsd_settings[:host]
+          @statsd = ::Statsd.new(statsd_settings[:host])
+          @statsd.port = statsd_settings[:port] if statsd_settings[:port]
+          @statsd.namespace = statsd_settings[:namespace] if statsd_settings[:namespace]
+          @statsd.batch_size = statsd_settings[:batch_size] if statsd_settings.key?(:batch_size)
+          @statsd.batch_byte_size = statsd_settings[:batch_byte_size] if statsd_settings.key?(:batch_byte_size)
+          @append_hostname = !statsd_settings.key?(:append_hostname) || !!statsd_settings[:append_hostname]
+        else
+          @statsd = nil
+        end
+      end
+      @statsd
+    end
+
+    def self.append_hostname?
+      @append_hostname
+    end
+
+    def self.reset_instance
+      remove_instance_variable(:@statsd) if defined?(@statsd)
+      Thread.current[:inst_statsd] = nil
+    end
+  end
+end