statsdserver 0.4

data/bin/statsd

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+
+require "rubygems"
+require "bundler/setup"
+
+require "parseconfig"
+require "statsdserver"
+require "statsdserver/output/stdout"
+require "statsdserver/output/tcp"
+require "sysexits"
+
+include Sysexits
+progname = File.basename($0)
+
+if ARGV.length != 1
+  $stderr.puts "usage: #{progname} path_to_config"
+  exit EX_USAGE
+end
+
+config_path = ARGV.shift
+begin
+  config_file = ParseConfig.new(config_path)
+rescue
+  $stderr.puts "#{progname}: error reading #{config_path}: #{$!}"
+  exit EX_DATAERR
+end
+
+config = {}
+%w(daemonize inputs flush_interval outputs prefix percentile
+   suffix).each do |key|
+  config[key.to_sym] = config_file[key] if config_file[key]
+end
+
+if config[:inputs].nil? || config[:inputs].empty?
+  $stderr.puts "#{progname}: no inputs specified"
+  exit EX_DATAERR
+end
+
+if config[:outputs].nil? || config[:outputs].empty?
+  $stderr.puts "#{progname}: no outputs specified"
+  exit EX_DATAERR
+end
+
+input_config = {}
+config[:inputs].split(/, */).each do |input|
+  input_config[input] = config_file["input:#{input}"] || {}
+end
+
+output_config = {}
+config[:outputs].split(/, */).each do |output|
+  output_config[output] = config_file["output:#{output}"] || {}
+end
+
+EM.run do
+  s = StatsdServer.new(config, input_config, output_config)
+  s.run
+end

data/lib/statsdserver.rb

@@ -0,0 +1,216 @@
+require "logger"
+require "statsdserver/input/udp"
+require "statsdserver/input/zeromq"
+require "statsdserver/stats"
+
+# Hack because the latest amqp gem uses String#bytesize, and not everyone
+# is running ruby 1.8.7.
+if !String.instance_methods.include?(:bytesize)
+  class String
+    alias :bytesize :length
+  end
+end
+
+class StatsdServer
+  attr_accessor :logger
+  attr_accessor :stats
+
+  public
+  def initialize(opts, input_config, output_config)
+    @stats = StatsdServer::Stats.new
+    @logger = Logger.new(STDERR)
+    @logger.progname = File.basename($0)
+
+    @opts = {
+      :bind => "127.0.0.1",
+      :port => 8125,
+      :percentile => 90,
+      :flush_interval => 30,
+      :prefix => "stats"
+    }.merge(opts)
+    @input_config = input_config
+    @output_config = output_config
+
+    # argument checking
+    [:port, :percentile, :flush_interval].each do |key|
+      begin
+        @opts[key] = Float(@opts[key])
+      rescue
+        raise "#{key}: #{@opts[key].inspect}: must be a valid number"
+      end
+    end
+  end # def initialize
+
+  public
+  def run
+    # initialize outputs
+    @outputs = []
+    @output_config.each do |output, config|
+      klass = StatsdServer::Output.const_get(output.capitalize)
+      if klass.nil?
+        @logger.fatal("unknown output #{output.inspect}")
+        exit EX_DATAERR
+      end
+      @outputs << klass.new(config)
+    end # @output_config.each
+
+    # start inputs
+    @input_config.each do |input, config|
+      case input
+      when "udp"
+        EM.open_datagram_socket(config["bind"], config["port"].to_i,
+                                Input::Udp) do |s|
+          s.logger = @logger
+          s.stats = @stats
+        end # EM.open_datagram_socket
+      when "zeromq"
+        s = Input::ZeroMQ.new
+        s.logger = @logger
+        s.stats = @stats
+        $ctx = EM::ZeroMQ::Context.new(1)
+        sock = $ctx.socket(ZMQ::PULL, s)
+        sock.setsockopt(ZMQ::HWM, 100)
+        sock.bind(config["bind"])
+      else
+        @logger.fatal("unknown input #{input.inspect}")
+        exit EX_DATAERR
+      end # case input
+    end # @inputs.each
+
+    # start flusher
+    Thread.abort_on_exception = true
+    @flusher = Thread.new do
+      while sleep(@opts[:flush_interval])
+        begin
+          flush
+        rescue => e
+          @logger.warn("trouble flushing: #{$!}")
+          @logger.debug(e.backtrace.join("\n"))
+        end
+      end
+    end
+  end # def run
+
+  #private
+  #def setup_amqp
+  #  begin
+  #    require "amqp"
+  #  rescue LoadError
+  #    @logger.fatal("missing amqp ruby module. try gem install amqp")
+  #    exit(1)
+  #  end
+#
+#    user = @output_url.user || ""
+#    user, vhost = user.split("@", 2)
+#    _, mqtype, mqname = @output_url.path.split("/", 3)
+#    amqp_settings = {
+#      :host => @output_url.host,
+#      :port => @output_url.port || 5672,
+#      :user => user,
+#      :pass => @output_url.password,
+#      :vhost => vhost || "/",
+#    }
+#
+#
+#    @amqp = AMQP.connect(amqp_settings)
+#    @channel = AMQP::Channel.new(@amqp)
+#
+#    opts = {
+#      :durable => true,
+#      :auto_delete => false,
+#    }
+#
+#    if @output_url.query
+#      @output_url.query.split("&").each do |param|
+#        k, v = param.split("=", 2)
+#        opts[:durable] = false if k == "durable" and v == "false"
+#        opts[:auto_delete] = true if k == "autodelete" and v == "true"
+#      end
+#    end
+#
+#    @exchange = case mqtype
+#    when "fanout"
+#      @channel.fanout(mqname, opts)
+#    when "direct"
+#      @channel.exchange(mqname, opts)
+#    when "topic"
+#      @channel.topic(mqname, opts)
+#    else
+#      raise TypeError, "unknown amqp output type #{mqtype}"
+#    end
+#  end # def setup_amqp
+
+  public
+  def carbon_update_str
+    updates = []
+    now = Time.now.to_i
+
+    timers = {}
+    @stats.timers.each do |k, v|
+      timers[k] = @stats.timers.delete(k)
+    end
+
+    counters = {}
+    @stats.counters.each do |k, v|
+      counters[k] = @stats.counters.delete(k)
+    end
+
+    timers.each do |key, values|
+      next if values.length == 0
+      values.sort!
+
+      # do some basic summarizing of our timer data
+      min = values[0]
+      max = values[-1]
+      mean = min
+      maxAtThreshold = min
+      if values.length > 1
+        threshold_index = ((100 - @opts[:percentile]) / 100.0) \
+                  * values.length
+        threshold_count = values.length - threshold_index.round
+        valid_values = values.slice(0, threshold_count)
+        maxAtThreshold = valid_values[-1]
+        sum = 0
+        valid_values.each { |v| sum += v }
+        mean = sum / valid_values.length
+      end
+
+      prefix = @opts[:prefix] ? "#{@opts[:prefix]}." : ""
+      suffix = @opts[:suffix] ? ".#{@opts[:suffix]}" : ""
+      updates << "#{prefix}timers.#{key}.mean#{suffix} #{mean} #{now}"
+      updates << "#{prefix}timers.#{key}.upper#{suffix} #{max} #{now}"
+      updates << "#{prefix}timers.#{key}.upper_#{@opts[:percentile]}#{suffix} " \
+            "#{maxAtThreshold} #{now}"
+      updates << "#{prefix}timers.#{key}.lower#{suffix} #{min} #{now}"
+      updates << "#{prefix}timers.#{key}.count#{suffix} #{values.length} #{now}"
+    end # timers.each
+
+    # Keep sending a 0 for counters (even if we don't get updates)
+    counters.keys.each do |k|
+      @stats.counters[k] ||= 0    # Keep sending a 0 if we don't get updates
+    end
+
+    counters.each do |key, value|
+      prefix = @opts[:prefix] ? "#{@opts[:prefix]}." : ""
+      suffix = @opts[:suffix] ? ".#{@opts[:suffix]}" : ""
+      updates << "#{prefix}#{key}#{suffix} #{value / @opts[:flush_interval]} #{now}"
+    end # counters.each
+
+    return updates.length == 0 ? nil : updates.join("\n") + "\n"
+  end # def carbon_update_str
+
+  public
+  def flush
+    s = carbon_update_str
+    return unless s
+
+    if @outputs.nil? or @outputs.length == 0
+      @logger.warn("no outputs configured, can't flush data")
+      return
+    end
+
+    @outputs.each do |output|
+      output.send(s)
+    end
+  end # def flush
+end # class StatsdServer

data/lib/statsdserver/input/udp.rb

@@ -0,0 +1,28 @@
+require "eventmachine"
+require "logger"
+require "statsdserver/proto/v1"
+
+class StatsdServer
+  class Input
+    class Udp < EventMachine::Connection
+      attr_accessor :logger,
+                    :stats
+
+      public
+      def initialize
+        @logger = Logger.new(STDOUT)
+      end
+
+      public
+      def receive_data(packet)
+        raise "@stats must be set" unless @stats
+
+        begin
+          StatsdServer::Proto::V1.parse(packet, @stats)
+        rescue StatsdServer::Proto::ParseError => e
+          @logger.warn(e.message)
+        end
+      end # def receive_data
+    end # class Ucp
+  end # class Input
+end # class StatsdServer

data/lib/statsdserver/input/zeromq.rb

@@ -0,0 +1,51 @@
+require "em-zeromq"
+require "logger"
+require "statsdserver/proto/v1"
+
+class StatsdServer
+  class Input
+    class ZeroMQ
+      attr_accessor :logger,
+                    :stats
+
+      public
+      def initialize
+        @logger = Logger.new(STDOUT)
+      end
+
+      public
+      def on_readable(socket, parts)
+        parts.each do |part|
+          str = part.copy_out_string
+          receive_data(str)
+        end
+      end
+
+      public
+      def receive_data(packet)
+        raise "@stats must be set" unless @stats
+
+        sep = packet.index(";")
+        if sep.nil?
+          @logger.warn("received unversioned update: #{packet}")
+          return
+        end
+
+        proto_ver = packet[0 .. sep - 1]
+        payload = packet[sep + 1 .. -1]
+        case proto_ver
+        when "1"
+          begin
+            StatsdServer::Proto::V1.parse(payload, @stats)
+          rescue StatsdServer::Proto::ParseError => e
+            @logger.warn(e.message)
+          end
+        else
+          @logger.warn("unknown protocol version #{proto_ver} in update #{packet}")
+          return
+        end
+
+      end # def receive_data
+    end # class Ucp
+  end # class Input
+end # class StatsdServer

data/lib/statsdserver/output/stdout.rb

@@ -0,0 +1,16 @@
+class StatsdServer::Output
+  class Stdout
+    attr_accessor :logger
+
+    public
+    def initialize(opts = {})
+      @logger = Logger.new(STDERR)
+      $stdout.sync = true   # autoflush
+    end
+
+    public
+    def send(str)
+      puts str
+    end
+  end # class Stdout
+end # class StatsdServer::Output

data/lib/statsdserver/output/tcp.rb

@@ -0,0 +1,33 @@
+require "logger"
+require "socket"
+
+class StatsdServer::Output
+  class Tcp
+    attr_accessor :logger
+
+    public
+    def initialize(opts = {})
+      if opts["host"].nil?
+        raise ArgumentError, "missing host in [output:tcp] config section"
+      end
+
+      if opts["port"].nil?
+        raise ArgumentError, "missing port in [output:tcp] config section"
+      end
+
+      @opts = opts
+      @logger = Logger.new(STDOUT)
+    end
+
+    public
+    def send(str)
+      @socket ||= connect
+      @socket.write("#{str}\n")
+    end
+
+    private
+    def connect
+      TCPSocket.new(@opts["host"], @opts["port"].to_i)
+    end
+  end # class Tcp
+end # class StatsdServer::Output

data/lib/statsdserver/proto/parseerror.rb

@@ -0,0 +1,6 @@
+class StatsdServer
+  class Proto
+    class ParseError < StandardError
+    end
+  end
+end

data/lib/statsdserver/proto/v1.rb

@@ -0,0 +1,64 @@
+require "statsdserver/proto/parseerror"
+
+class StatsdServer
+  class Proto
+    module V1
+      def self.parse(data, stats)
+        data.split("\n").each do |update|
+          self.parse_update(update, stats)
+        end
+      end # def parse
+
+      def self.parse_update(update, stats)
+        bits = update.split(":")
+        # TODO: optimize into single regexp & compile?
+        key = bits.shift.gsub(/\s+/, "_") \
+                .gsub(/\//, "-") \
+                .gsub(/[^a-zA-Z_\-0-9\.]/, "")
+        bits << "1" if bits.length == 0
+        bits.each do |bit|
+          fields = bit.split("|")
+          if fields.length != 2
+            raise ParseError, "invalid update: #{bit}"
+          end
+
+          if fields[1] == "ms" # timer update
+            if fields[0].index(",")
+              fields[0].split(",").each do |value_str|
+                value = Integer(value_str) rescue nil
+                stats.timers[key] << value if value
+              end
+            else
+              value = Integer(fields[0]) rescue nil
+              if value.nil?
+                raise ParseError, "invalid timer value: #{fields[0]}"
+              end
+              stats.timers[key] << fields[0].to_i
+            end
+          elsif fields[1] == "c" # counter update
+            count_str, sample_rate_str = fields[0].split("@", 2)
+
+            if sample_rate_str
+              sample_rate = Float(sample_rate_str) rescue nil
+              if sample_rate.nil?
+                raise ParseError, "invalid sample_rate: #{sample_rate_str}"
+              end
+            else
+              sample_rate = 1
+            end
+
+            count = Integer(count_str) rescue nil
+            if count.nil?
+              raise ParseError, "invalid count: #{count_str}"
+            end
+
+            stats.counters[key] += count.to_i * (1 / sample_rate.to_f)
+          else
+            raise ParseError,
+                  "invalid update: #{update}: unknown type #{fields[1]}"
+          end
+        end
+      end # def parse_update
+    end # module V1
+  end # class Proto
+end # class StatsdServer

data/lib/statsdserver/stats.rb

@@ -0,0 +1,16 @@
+require "logger"
+
+class StatsdServer
+  class Stats
+    attr_accessor :counters,
+                  :timers,
+                  :logger
+
+    public
+    def initialize
+      @timers = Hash.new { |h, k| h[k] = Array.new }
+      @counters = Hash.new { |h, k| h[k] = 0 }
+      @logger = Logger.new(STDERR)
+    end
+  end # class Stats
+end # class StatsdServer

metadata

@@ -0,0 +1,168 @@
+--- !ruby/object:Gem::Specification
+name: statsdserver
+version: !ruby/object:Gem::Version
+  version: '0.4'
+  prerelease: 
+platform: ruby
+authors:
+- Pete Fritchman
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-08-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: amqp
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: parseconfig
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sysexits
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: em-zeromq
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: collect and aggregate stats, flush to graphite
+email: petef@databits.net
+executables:
+- statsd
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/statsdserver/output/tcp.rb
+- lib/statsdserver/output/stdout.rb
+- lib/statsdserver/input/zeromq.rb
+- lib/statsdserver/input/udp.rb
+- lib/statsdserver/proto/v1.rb
+- lib/statsdserver/proto/parseerror.rb
+- lib/statsdserver/stats.rb
+- lib/statsdserver.rb
+- bin/statsd
+homepage: https://github.com/fetep/ruby-statsd
+licenses:
+- Apache License 2.0
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: statsd (server) -- stat collector/aggregator
+test_files: []