statsdserver 0.4 → 0.5

data/bin/statsd

@@ -1,13 +1,15 @@
 #!/usr/bin/env ruby
 
+$: << File.join(File.dirname(__FILE__), "..", "lib")
 
 require "rubygems"
-require "bundler/setup"
 
+require "daemons"
 require "parseconfig"
 require "statsdserver"
-require "statsdserver/output/stdout"
+require "statsdserver/output/amqp"
 require "statsdserver/output/tcp"
+require "statsdserver/output/stdout"
 require "sysexits"
 
 include Sysexits
@@ -28,7 +30,7 @@ end
 
 config = {}
 %w(daemonize inputs flush_interval outputs prefix percentile
-   suffix).each do |key|
+   suffix preserve_counters).each do |key|
   config[key.to_sym] = config_file[key] if config_file[key]
 end
 
@@ -42,16 +44,26 @@ if config[:outputs].nil? || config[:outputs].empty?
   exit EX_DATAERR
 end
 
+config[:inputs] = config[:inputs].split(/, */)
 input_config = {}
-config[:inputs].split(/, */).each do |input|
+config[:inputs].each do |input|
   input_config[input] = config_file["input:#{input}"] || {}
 end
 
+config[:outputs] = config[:outputs].split(/, */)
 output_config = {}
-config[:outputs].split(/, */).each do |output|
+config[:outputs].each do |output|
   output_config[output] = config_file["output:#{output}"] || {}
 end
 
+if config_file["daemonize"] == "true"
+  if config[:outputs].include?("stdout")
+    $stderr.puts "#{progname}: output stdout is not compatible with daemonize"
+    exit EX_DATAERR
+  end
+  Daemons.daemonize(:app_name => progname)
+end
+
 EM.run do
   s = StatsdServer.new(config, input_config, output_config)
   s.run

data/lib/statsdserver.rb

@@ -1,6 +1,7 @@
 require "logger"
 require "statsdserver/input/udp"
 require "statsdserver/input/zeromq"
+require "statsdserver/math"
 require "statsdserver/stats"
 
 # Hack because the latest amqp gem uses String#bytesize, and not everyone
@@ -26,7 +27,8 @@ class StatsdServer
       :port => 8125,
       :percentile => 90,
       :flush_interval => 30,
-      :prefix => "stats"
+      :prefix => "stats",
+      :preserve_counters => "true",
     }.merge(opts)
     @input_config = input_config
     @output_config = output_config
@@ -140,60 +142,65 @@ class StatsdServer
 #    end
 #  end # def setup_amqp
 
+  private
+  def metric_name(name)
+    if @opts[:prefix] && !@opts[:prefix].empty?
+      prefix = @opts[:prefix] + "."
+    else
+      prefix = ""
+    end
+
+    if @opts[:suffix] && !@opts[:suffix].empty?
+      suffix = @opts[:suffix] + "."
+    else
+      suffix = ""
+    end
+
+    return [prefix, name, suffix].join("")
+  end
+
   public
   def carbon_update_str
     updates = []
     now = Time.now.to_i
 
     timers = {}
-    @stats.timers.each do |k, v|
+    @stats.timers.keys.each do |k|
       timers[k] = @stats.timers.delete(k)
     end
 
     counters = {}
-    @stats.counters.each do |k, v|
+    @stats.counters.keys.each do |k|
       counters[k] = @stats.counters.delete(k)
     end
 
-    timers.each do |key, values|
-      next if values.length == 0
-      values.sort!
-
-      # do some basic summarizing of our timer data
-      min = values[0]
-      max = values[-1]
-      mean = min
-      maxAtThreshold = min
-      if values.length > 1
-        threshold_index = ((100 - @opts[:percentile]) / 100.0) \
-                  * values.length
-        threshold_count = values.length - threshold_index.round
-        valid_values = values.slice(0, threshold_count)
-        maxAtThreshold = valid_values[-1]
-        sum = 0
-        valid_values.each { |v| sum += v }
-        mean = sum / valid_values.length
+    if @opts[:preserve_counters] == "true"
+      # Keep sending a 0 for counters (even if we don't get updates)
+      counters.keys.each do |k|
+        @stats.counters[k] ||= 0    # Keep sending a 0 if we don't get updates
       end
+    end
 
-      prefix = @opts[:prefix] ? "#{@opts[:prefix]}." : ""
-      suffix = @opts[:suffix] ? ".#{@opts[:suffix]}" : ""
-      updates << "#{prefix}timers.#{key}.mean#{suffix} #{mean} #{now}"
-      updates << "#{prefix}timers.#{key}.upper#{suffix} #{max} #{now}"
-      updates << "#{prefix}timers.#{key}.upper_#{@opts[:percentile]}#{suffix} " \
-            "#{maxAtThreshold} #{now}"
-      updates << "#{prefix}timers.#{key}.lower#{suffix} #{min} #{now}"
-      updates << "#{prefix}timers.#{key}.count#{suffix} #{values.length} #{now}"
+    timers.each do |key, values|
+      next if values.length == 0
+      summary = ::StatsdServer::Math.summarize(values, @opts)
+
+      updates << [metric_name("timers.#{key}.mean"),
+                  summary[:mean], now].join(" ")
+      updates << [metric_name("timers.#{key}.upper"),
+                  summary[:upper], now].join(" ")
+      updates << [metric_name("timers.#{key}.lower"),
+                  summary[:lower], now].join(" ")
+      updates << [metric_name("timers.#{key}.count"),
+                  values.length, now].join(" ")
+      updates << [metric_name("timers.#{key}.upper_#{@opts[:percentile]}"),
+                  summary[:max_at_threshold], now].join(" ")
     end # timers.each
 
-    # Keep sending a 0 for counters (even if we don't get updates)
-    counters.keys.each do |k|
-      @stats.counters[k] ||= 0    # Keep sending a 0 if we don't get updates
-    end
-
     counters.each do |key, value|
-      prefix = @opts[:prefix] ? "#{@opts[:prefix]}." : ""
-      suffix = @opts[:suffix] ? ".#{@opts[:suffix]}" : ""
-      updates << "#{prefix}#{key}#{suffix} #{value / @opts[:flush_interval]} #{now}"
+      updates << [metric_name(key),
+                  value / @opts[:flush_interval],
+                  now].join(" ")
     end # counters.each
 
     return updates.length == 0 ? nil : updates.join("\n") + "\n"

data/lib/statsdserver/input/zeromq.rb

@@ -1,4 +1,3 @@
-require "em-zeromq"
 require "logger"
 require "statsdserver/proto/v1"
 
@@ -10,6 +9,16 @@ class StatsdServer
 
       public
       def initialize
+        begin
+          require "em-zeromq"
+        rescue LoadError => e
+          raise unless e.message =~ /em-zeromq/
+          new_e = \
+            e.exception("Please install the em-zeromq gem for ZeroMQ input.")
+          new_e.set_backtrace(e.backtrace)
+          raise new_e
+        end
+
         @logger = Logger.new(STDOUT)
       end
 

data/lib/statsdserver/math.rb

@@ -0,0 +1,29 @@
+require "logger"
+
+class StatsdServer
+  module Math
+    def self.summarize(values, opts = {})
+      opts = {
+        :percentile => 90,
+      }.merge(opts)
+      res = {}
+
+      values.sort!
+      res[:min] = values[0]
+      res[:max] = values[-1]
+      res[:mean] = res[:min]
+      res[:max_at_threshold] = res[:min]
+      if values.length > 1
+        threshold_index = ((100 - opts[:percentile]) / 100.0) * values.length
+        threshold_count = values.length - threshold_index.round
+        valid_values = values.slice(0, threshold_count)
+        res[:max_at_threshold] = valid_values[-1]
+        sum = 0
+        valid_values.each { |v| sum += v }
+        res[:mean] = sum / valid_values.length
+      end
+
+      return res
+    end
+  end # class Math
+end # class StatsdServer

data/lib/statsdserver/output/amqp.rb

@@ -0,0 +1,60 @@
+require "logger"
+
+class StatsdServer::Output
+  class Amqp
+    attr_accessor :logger
+
+    public
+    def initialize(opts = {})
+      begin
+        require "bunny"
+      rescue LoadError => e
+        raise unless e.message =~ /bunny/
+        new_e = e.exception("Please install the bunny gem for AMQP output.")
+        new_e.set_backtrace(e.backtrace)
+        raise new_e
+      end
+
+      if opts["exchange_type"].nil?
+        raise ArgumentError, "missing host in [output:tcp] config section"
+      end
+
+      if opts["exchange_name"].nil?
+        raise ArgumentError, "missing port in [output:tcp] config section"
+      end
+
+      @opts = opts
+      @logger = Logger.new(STDOUT)
+    end
+
+    public
+    def send(str)
+      if @bunny.nil?
+        @bunny, @exchange = connect
+      end
+
+      begin
+        @exchange.publish(str)
+      rescue => e
+        @bunny.close_connection rescue nil
+        @bunny = nil
+        raise
+      end
+    end
+
+    private
+    def connect
+      bunny = Bunny.new(@opts)
+      bunny.start
+
+      exchange = bunny.exchange(
+        @opts["exchange_name"],
+        :type => @opts["exchange_type"].to_sym,
+        :durable => @opts["exchange_durable"] == "true" ? true : false,
+        :auto_delete => @opts["exchange_auto_delete"] == "true" ? true : false,
+      )
+
+      return bunny, exchange
+    end
+  end # class Amqp
+end # class StatsdServer::Output

data/lib/statsdserver/output/tcp.rb

@@ -22,7 +22,14 @@ class StatsdServer::Output
     public
     def send(str)
       @socket ||= connect
-      @socket.write("#{str}\n")
+      begin
+        @socket.write("#{str}")
+      rescue => e
+        # set @socket to nil to force a re-connect, then pass up the exception
+        @socket.close rescue nil
+        @socket = nil
+        raise
+      end
     end
 
     private

data/lib/statsdserver/proto/v1.rb

@@ -22,7 +22,7 @@ class StatsdServer
             raise ParseError, "invalid update: #{bit}"
           end
 
-          if fields[1] == "ms" # timer update
+          if fields[1] == "ms" or fields[1] == "t" # timer update
             if fields[0].index(",")
               fields[0].split(",").each do |value_str|
                 value = Integer(value_str) rescue nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: statsdserver
 version: !ruby/object:Gem::Version
-  version: '0.4'
+  version: '0.5'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-08 00:00:00.000000000 Z
+date: 2012-08-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -44,7 +44,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: amqp
+  name: daemons
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -107,6 +107,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: em-zeromq
   requirement: !ruby/object:Gem::Requirement
@@ -115,7 +131,7 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
@@ -130,13 +146,15 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/statsdserver/output/tcp.rb
-- lib/statsdserver/output/stdout.rb
-- lib/statsdserver/input/zeromq.rb
+- lib/statsdserver/math.rb
 - lib/statsdserver/input/udp.rb
-- lib/statsdserver/proto/v1.rb
-- lib/statsdserver/proto/parseerror.rb
+- lib/statsdserver/input/zeromq.rb
+- lib/statsdserver/output/stdout.rb
+- lib/statsdserver/output/tcp.rb
+- lib/statsdserver/output/amqp.rb
 - lib/statsdserver/stats.rb
+- lib/statsdserver/proto/parseerror.rb
+- lib/statsdserver/proto/v1.rb
 - lib/statsdserver.rb
 - bin/statsd
 homepage: https://github.com/fetep/ruby-statsd