afstatsd 0.0.1

data/example/example.rb

@@ -0,0 +1,69 @@
+require 'afstatsd'
+
+#$statsd = Statsd.new 'statsd_server.my_company.com', 8125, 20
+
+$statsd = Statsd.new     # use defaults
+$statsd.namespace = 'test.ruby'
+
+
+$statsd.increment 'counter1'
+$statsd.increment 'counter1'
+$statsd.decrement 'counter1'        #counters accumulate
+
+$statsd.gauge 'gauge1', 1024
+$statsd.gauge 'gauge1', 1025
+$statsd.gauge 'gauge1', 1026
+$statsd.gauge 'gauge1', 1027      
+$statsd.gauge 'gauge1', 1028        # gauges get averaged when aggregated
+
+$statsd.time('timing1' ){sleep 0.01}        
+$statsd.time('timing1' ){sleep 0.02}
+$statsd.time('timing1' ){sleep 0.03}
+$statsd.time('timing1' ){sleep 0.04}    # timings get averaged when aggregated
+
+
+=begin
+
+100.times do 
+    #$statsd.increment 'sampled', 0.1, 'sampled'
+    $statsd.increment 'sampled'
+end
+
+$statsd.set 'set1', 1099, "ez"    
+
+for i in 10..19 do
+    $statsd.increment "counter#{i}"  # create a group of counters
+end    
+
+1000.times do 
+    $statsd.increment 'fast'            # don't do this if aggregation is off
+end    
+
+# In this test program, this will give the aggregator time to run.
+15.times do
+    sleep 2
+    $statsd.increment 'slow'             
+end
+
+=end
+
+=begin
+# test for thread safety
+threads = []
+start = Time.now
+for i in 0..9 do
+    threads << Thread.new(i) do |j| 
+        start = Time.now
+        1000000.times do
+            $statsd.increment 'inthethread'
+#            sleep(0.01)
+        end
+        puts "thread #{j} says: I took #{((Time.now - start)*1000).round} ms"
+    end
+end
+threads.each { |t| t.join }
+
+puts "total time: #{((Time.now - start)*1000).round} ms"
+=end
+
+puts "#{$statsd.dropped} messages dropped"

data/lib/afstatsd/statsd_aggregator.rb

@@ -0,0 +1,132 @@
+# Statsd Aggregator
+#
+# Used to aggregate metrics in a threaded environment.  Only one of these 
+# should be created, in the main thread.
+# For each thread, we create 2 buffers.  The thread will be writing to 
+# one, while the aggregator reads from the other.  The aggregator will
+# control which set is which.
+  
+  
+ class StatsdAggregator 
+    attr_accessor :transport
+
+    def initialize(interval=20)
+        @interval = interval
+        @timer = nil
+        @mutex = Mutex.new    
+        @running = false
+        @left_buffers = {}        # 2 buffer groups
+        @right_buffers = {}       #   each buffer group is a hash
+        @rbufs = @left_buffers    # buffer group currently being read from
+        @wbufs = @right_buffers   # buffer group currently being written to
+        at_exit do
+            if @running
+                flush_buffers 
+                swap_buffers
+                flush_buffers
+            end 
+        end
+    end    
+
+    def start(transport)
+        @transport = transport
+        return if @running  # already started
+        # Spin up a thread to periodically send the aggregated stats.
+        # Divide the interval in half to allow other threads to finish
+        # their writes after we swap, and before we start reading.
+        @timer = Thread.new do
+            loop do
+                sleep @interval/2
+                swap_buffers
+                sleep @interval/2
+                flush_buffers
+            end
+        end
+        @running = true
+        #puts "aggregation started.  Interval=#{@interval}"
+    end
+    
+    def stop
+        return if not @running    # already stopped
+        flush_buffers                    
+        @timer.kill if @timer
+        @timer = nil
+        @running = false
+        #puts "aggregation stopped"
+    end
+  
+    def set_interval(interval)
+        @interval = interval
+    end
+    
+    # the following methods are thread safe
+    
+    def running
+        @running
+    end
+
+    # this is the only method that should be used by child threads.
+    def add(metric)
+        # We should have a write buffer assigned to our thread.  
+        # Create one if not.
+        unless write_buffer = @wbufs[Thread.current]
+            #puts "Thread #{Thread.current}: creating write_buffer"
+            write_buffer = {}
+            # get a lock before we mess with the global hash
+            @mutex.synchronize do
+                @wbufs[Thread.current] = write_buffer
+            end
+        end
+        if m = write_buffer[metric.name]
+            # if we are already collecting this metric, just aggregate the new value
+            m.aggregate metric.value
+        else
+            # otherwise, add this metric to the aggregation buffer
+            #puts "Thread #{Thread.current}: creating metric"
+            write_buffer[metric.name] = metric
+        end
+        #puts "Thread #{Thread.current}: Added metric: #{metric}"
+    end
+    
+    private
+    
+    # Next two methods are called at different times during the interval,
+    # so any writes in progress after the swap will have time to complete.
+
+    def swap_buffers 
+        if @rbufs == @left_buffers
+            @rbufs = @right_buffers
+            @wbufs = @left_buffers
+        else
+            @rbufs = @left_buffers
+            @wbufs = @right_buffers
+        end
+    end
+    
+    def flush_buffers
+        # Each thread has it's own read buffer.  If it's empty, the
+        # thread might be dead.  We'll delete it's read buffer.
+        @rbufs.delete_if { |k, rb| rb.empty? }
+        
+        # If not empty, aggregate all the data across all the threads, 
+        # then send.
+        send_buffer = {}
+        @rbufs.each_value do |rb|
+            rb.each_value do |metric|
+                if m = send_buffer[metric.name]
+                    m.aggregate metric.value
+                else    
+                    send_buffer[metric.name] = metric
+                end    
+            end
+            # once we've aggregated all the metrics from this 
+            # thread, clear out the buffer, but don't remove it.        
+            rb.clear
+        end    
+        #puts "nothing to send" if send_buffer.empty? 
+        send_buffer.each_value do |metric|
+            @transport.call(metric)
+        end    
+    end
+    
+end    # class StatsdAggregator

data/lib/afstatsd/statsd_metrics.rb

@@ -0,0 +1,106 @@
+# Classes used to store and manipulate each type of metric
+# each type must implement initialize, aggregate, and to_s
+
+module StatsdMetrics
+
+class Metric
+    # all metrics share these
+    attr_accessor :name
+    attr_accessor :value
+    attr_accessor :message
+end
+
+class CMetric < Metric
+    # Counter
+    def initialize(name, value, rate=1, msg="")
+        @name = name
+        @value = value
+        @message = msg
+        @sample_rate = rate
+    end
+    
+    def aggregate(delta)
+        @value += delta        #accumulate
+    end
+    
+    def to_s
+        if @sample_rate == 1 then r = "" else r = "|@#{@sample_rate}" end    
+        if @message == "" 
+            m = "" 
+        else 
+            if r == "" 
+                m = "||#{@message}" 
+            else    
+                m = "|#{@message}" 
+            end
+        end
+        "#{name}:#{@value}|c#{r}#{m}"
+    end
+end
+
+class GMetric < Metric
+    # Guage
+    def initialize(name, value, msg="")
+        @name = name
+        @value = value
+        @message = msg
+        @count = 1
+    end
+    
+    def aggregate(value)
+        @value += value        #average
+        @count += 1
+    end
+    
+    def to_s
+        avg = @value / @count
+        if @message == "" then m = "" else m = "|#{@message}" end
+        "#{name}:#{avg}|g#{m}"
+    end
+
+end
+
+class TMetric < Metric
+    # Timing
+    def initialize(name, value, rate=1, msg="")
+        @name = name
+        @value = value
+        @sample_rate = rate
+        @message = msg
+        @count = 1
+    end
+    
+    def aggregate(value)
+        @value += value        #average
+        @count += 1
+    end
+    
+    def to_s
+        avg = @value / @count
+        if @message == "" then m = "" else m = "|#{@message}" end
+        "#{name}:#{avg}|ms#{m}"
+    end
+
+end
+
+class SMetric < Metric
+    # Set (per the etsy standard)
+    def initialize(name, value, msg="")
+        @name = name
+        @value = value
+        @message = msg
+    end
+    
+    def aggregate(value)
+        @value = value        #overwrite
+    end
+    
+    def to_s
+        if @message == "" then m = "" else m = "|#{@message}" end
+        "#{name}:#{@value}|s#{m}"
+    end
+
+end
+
+end     #module StatsdMetrics
+

data/lib/afstatsd.rb

@@ -0,0 +1,281 @@
+require 'socket'
+require 'forwardable'
+require 'rubygems'
+require 'posix_mq'
+require 'afstatsd/statsd_metrics'
+require 'afstatsd/statsd_aggregator'
+require 'monitor'
+require 'fcntl'
+
+# = Statsd: A Statsd client (https://github.com/etsy/statsd)
+#
+# @example Set up a global Statsd client for a server on localhost:9125, 
+#                                aggregate 20 seconds worth of metrics
+#   $statsd = Statsd.new 'localhost', 8125, 20
+# @example Send some stats
+#   $statsd.increment 'garets'
+#   $statsd.timing 'glork', 320
+#   $statsd.gauge 'bork', 100
+# @example Use {#time} to time the execution of a block
+#   $statsd.time('account.activate') { @account.activate! }
+# @example Create a namespaced statsd client and increment 'account.activate'
+#   statsd = Statsd.new('localhost').tap{|sd| sd.namespace = 'account'}
+#   statsd.increment 'activate'
+#
+# Statsd instances are thread safe for general usage, by using a thread local
+# UDPSocket and carrying no state. The attributes are stateful, and are not
+# mutexed, it is expected that users will not change these at runtime in
+# threaded environments. If users require such use cases, it is recommend that
+# users either mutex around their Statsd object, or create separate objects for
+# each namespace / host+port combination.
+class Statsd
+
+    # A namespace to prepend to all statsd calls.
+    attr_reader :namespace
+
+    # StatsD host. Defaults to 127.0.0.1.  Only used with UDP transport
+    attr_reader :host
+
+    # StatsD port. Defaults to 8125.  Only used with UDP transport
+    attr_reader :port
+
+    # StatsD namespace prefix, generated from #namespace
+    attr_reader :prefix
+
+    # a postfix to append to all metrics
+    attr_reader :postfix 
+
+    # count of messages that were dropped due to transmit error
+    attr_reader :dropped
+
+    class << self
+        # Set to a standard logger instance to enable debug logging.
+        attr_accessor :logger
+    end
+
+    # @param [String] host your statsd host
+    # @param [Integer] port your statsd port
+    # @param [Integer] interval for aggregatore
+    def initialize(host = '127.0.0.1', port = 8125, interval = 20)
+        self.host, self.port = host, port
+        @prefix = nil
+        @postfix = nil
+        @aggregator = StatsdAggregator.new(interval)
+        set_transport :mq_transport
+        self.aggregating = true unless interval == 0
+        @dropped = 0
+    end
+
+    # @param [method] The ruby symbol for the method that gets called to send
+    # one metric to the server.  eg: set_transport :udp_transport
+    def set_transport(transport)
+        @transport = method(transport)
+        @aggregator.transport = @transport  # aggregator needs to know
+    end
+
+    # @param [Boolean] Turn aggregation on or off
+    def aggregating= (should_aggregate)
+        if should_aggregate
+            @aggregator.start(@transport)
+        else
+            @aggregator.stop
+        end
+    end
+
+    # is the aggregator running?
+    def aggregating
+        @aggregator.running
+    end
+
+    # @attribute [w] namespace
+    #   Writes are not thread safe.
+    def namespace=(namespace)
+        @namespace = namespace
+        @prefix = "#{namespace}."
+    end
+
+    # @attribute [w] postfix
+    #   A value to be appended to the stat name after a '.'. If the value is
+    #   blank then the postfix will be reset to nil (rather than to '.').
+    def postfix=(pf)
+        case pf
+        when nil, false, '' then @postfix = nil
+        else @postfix = ".#{pf}"
+        end
+    end
+
+    # @attribute [w] host
+    #   Writes are not thread safe.
+    def host=(host)
+        @host = host || '127.0.0.1'
+    end
+
+    # @attribute [w] port
+    #   Writes are not thread safe.
+    def port=(port)
+        @port = port || 8125
+    end
+
+    # Sends an increment (count = 1) for the given stat to the statsd server.
+    #
+    # @param [String] stat stat name
+    # @param [Numeric] sample_rate sample rate, 1 for always
+    # @param [String] optional note (AppFirst extension to StatsD)
+    # @see #count
+    def increment(stat, sample_rate=1, note="")
+        count stat, 1, sample_rate, note
+    end
+
+    # Sends a decrement (count = -1) for the given stat to the statsd server.
+    #
+    # @param [String] stat stat name
+    # @param [Numeric] sample_rate sample rate, 1 for always
+    # @param [String] optional note (AppFirst extension to StatsD)
+    # @see #count
+    def decrement(stat, sample_rate=1, note="")
+        count stat, -1, sample_rate, note
+    end
+
+    # Sends an arbitrary count for the given stat to the statsd server.
+    #
+    # @param [String] stat stat name
+    # @param [Integer] count count
+    # @param [Numeric] sample_rate sample rate, 1 for always
+    # @param [String] optional note (AppFirst extension to StatsD)
+    def count(stat, count, sample_rate=1, note="")
+        if sample_rate == 1 or rand < sample_rate
+            send_metric StatsdMetrics::CMetric.new(expand_name(stat), count, sample_rate, note)
+        end
+    end
+
+    # Sends an arbitary gauge value for the given stat to the statsd server.
+    #
+    # This is useful for recording things like available disk space,
+    # memory usage, and the like, which have different semantics than
+    # counters.
+    #
+    # @param [String] stat stat name.
+    # @param [Numeric] value gauge value.
+    # @param [String] optional note (AppFirst extension to StatsD)
+    # @example Report the current user count:
+    #   $statsd.gauge('user.count', User.count)
+    def gauge(stat, value, note="")
+        send_metric StatsdMetrics::GMetric.new(expand_name(stat), value, note)
+    end
+
+    # Sends an arbitary set value for the given stat to the statsd server.
+    #
+    # This is for recording counts of unique events, which are useful to
+    # see on graphs to correlate to other values.  For example, a deployment
+    # might get recorded as a set, and be drawn as annotations on a CPU history
+    # graph.
+    #
+    # @param [String] stat stat name.
+    # @param [Numeric] value event value.
+    # @param [String] optional note (AppFirst extension to StatsD)
+    # @example Report a deployment happening:
+    #   $statsd.set('deployment', DEPLOYMENT_EVENT_CODE)
+    def set(stat, value, note="")
+        send_metric StatsdMetrics::SMetric.new(expand_name(stat), value, note)
+    end
+
+    # Sends a timing (in ms) for the given stat to the statsd server. The
+    # sample_rate determines what percentage of the time this report is sent. The
+    # statsd server then uses the sample_rate to correctly track the average
+    # timing for the stat.
+    #
+    # @param [String] stat stat name
+    # @param [Integer] ms timing in milliseconds
+    # @param [Numeric] sample_rate sample rate, 1 for always
+    # @param [String] optional note (AppFirst extension to StatsD)
+    def timing(stat, ms, sample_rate=1, note="")
+        if sample_rate == 1 or rand < sample_rate
+            send_metric StatsdMetrics::TMetric.new(expand_name(stat), ms, sample_rate, note)
+        end    
+    end
+
+    # Reports execution time of the provided block using {#timing}.
+    #
+    # @param [String] stat stat name
+    # @param [Numeric] sample_rate sample rate, 1 for always
+    # @param [String] optional note (AppFirst extension to StatsD)
+    # @yield The operation to be timed
+    # @see #timing
+    # @example Report the time (in ms) taken to activate an account
+    #   $statsd.time('account.activate') { @account.activate! }
+    def time(stat, sample_rate=1, note="")
+        start = Time.now
+        result = yield
+        timing(stat, ((Time.now - start) * 1000).round, sample_rate, note)
+        result
+    end
+
+    protected
+
+    def send_metric(metric)
+        # All the metric types above funnel to here.  We will send or aggregate.
+        if aggregating
+            @aggregator.add metric
+        else 
+            @transport.call(metric)
+        end
+    end
+
+    def expand_name(name)
+        # Replace Ruby module scoping with '.' and reserved chars (: | @) with underscores.
+        name = name.to_s.gsub('::', '.').tr(':|@', '_') 
+        "#{prefix}#{name}#{postfix}"
+    end
+
+    def udp_transport(metric)
+        #puts "socket < #{metric}\n"
+        self.class.logger.debug { "Statsd: #{metric}" } if self.class.logger
+        socket.send(metric.to_s, 0, @host, @port)
+        rescue => boom
+            #puts "socket send error"
+            @dropped +=1
+            self.class.logger.error { "Statsd: #{boom.class} #{boom}" } if self.class.logger
+            nil
+    end
+
+    STATSD_SEVERITY = 3
+    def mq_transport(metric)
+        #puts "MQ < #{metric}\n" #debug
+        self.class.logger.debug { "Statsd: #{metric}" } if self.class.logger
+        if not @mq 
+            begin
+                @mq = POSIX_MQ.new("/afcollectorapi", Fcntl::O_WRONLY | Fcntl::O_NONBLOCK)
+            rescue => boom
+                self.class.logger.debug { "Statsd: MQ open error #{boom.class} #{boom}" } if self.class.logger
+                # failed to open MQ.  Fall back to UPD transport.  Note:  Current message will be lost.
+                @dropped += 1
+                # puts "fallback to udp"
+                set_transport :udp_transport
+                return nil
+            end    
+        end
+        begin
+            @mq.send(metric.to_s, STATSD_SEVERITY)
+        rescue => boom
+            # just drop it on the floor
+            @dropped += 1
+            #puts "MQ send error: #{boom.class} #{boom}"
+            self.class.logger.error { "Statsd: MQ Send Error#{boom.class} #{boom}" } if self.class.logger
+            nil
+        end    
+    end
+
+    def both_transport(metric)
+        mq_transport(metric)
+        udp_transport(metric)
+    end
+    
+    private
+
+    def socket
+        Thread.current[:statsd_socket] ||= UDPSocket.new
+    end
+    
+end     # class Statsd
+
+

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification
+name: afstatsd
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Clark Bremer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: posix_mq
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+description: A StatsD library with AppFirst Extensions
+email: clark@appfirst.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/afstatsd.rb
+- lib/afstatsd/statsd_aggregator.rb
+- lib/afstatsd/statsd_metrics.rb
+- example/example.rb
+homepage: http://appfirst.com
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: AppFirst StatsD Library
+test_files: []
+has_rdoc: