petef-statsd 0.3

data/bin/statsd

@@ -0,0 +1,109 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "ostruct"
+require "optparse"
+require "statsd"
+
+FLUSH_INTERVAL = 10
+
+progname = File.basename($0)
+options = OpenStruct.new
+options.output_url = nil
+options.daemonize = false
+options.port = 8125
+options.bind = "0.0.0.0"
+options.pct_threshold = 90
+options.flush_interval = 10
+options.key_suffix = nil
+
+opts = OptionParser.new do |opts|
+  opts.banner = "#{progname} [-d] [-p pct] [-i sec] [-b [host:]port] [-s suffix] -o dest"
+
+  opts.separator "Required flags:"
+  opts.on("-o", "--output=url", String,
+          "Destination for aggregate data") do |x|
+    options.output_url = x
+  end
+
+  opts.separator "   output dest should be a URL, example:"
+  opts.separator "    * stdout:///"
+  opts.separator "    * tcp://graphite.host.com:2003"
+  opts.separator "    * amqp://user@vhost:pass@amqp.host.com/type/name"
+  opts.separator "      (where type is fanout, queue, or topic)"
+
+  opts.separator ""
+  opts.separator "Optional flags:"
+
+  opts.on("-d", "--[no-]daemonize",
+          "Detach from tty and fork (def false)") do |x|
+    raise NotImplementedError, "--daemonize not implemented yet"
+    #options.daemonize = x
+  end
+
+  opts.on("-b", "--bind=host:port", String,
+          "host:port to bind to for UDP listener (default 0.0.0.0:8125)") do |x|
+    host, port = x.split(":", 2)
+    if ! port  # just given a port
+      port = host
+      host = "0.0.0.0"
+    end
+    options.bind = host
+    options.port = port.to_i
+  end
+
+  opts.on("-p", "--percent=threshold", Integer,
+          "% threshold for calculating mean (def 90)") do |x|
+    options.pct_threshold = x
+  end
+
+  opts.on("-i", "--interval=seconds", Integer,
+          "Flush interval in seconds (def 10)") do |x|
+    options.flush_interval = x
+  end
+
+  opts.on("-s", "--suffix=string", String,
+          "suffix to append to all key names") do |x|
+    options.key_suffix = x
+  end
+end
+
+begin
+  opts.parse!
+  raise "must specify output (-o or --output)" unless options.output_url
+  if options.pct_threshold <= 0 or options.pct_threshold > 100
+    raise "pct_threshold #{options.pct_threshold} out of range, must be 1-100"
+  end
+  if options.port <= 0 or options.pct_threshold > 2**32
+    raise "bind port #{options.port} out of range"
+  end
+rescue
+  $stderr.puts "#{progname}: #{$!}"
+  $stderr.puts opts
+  exit(1)
+end
+
+EM.run do
+  begin
+    StatsD.pct_threshold = options.pct_threshold
+    StatsD.flush_interval = options.flush_interval
+    StatsD.output_url = options.output_url
+    StatsD.key_suffix = options.key_suffix
+
+    EM.add_periodic_timer(options.flush_interval) do
+      EM.defer do
+        begin
+          StatsD.flush
+        rescue
+          StatsD.logger.warn("trouble flushing: #{$!}")
+        end
+      end
+    end
+
+    EM.open_datagram_socket(options.bind, options.port, StatsD)
+  rescue
+    $stderr.puts "Exception inside of EM.run: #{$!}"
+    EM.stop_event_loop
+    exit 1
+  end
+end

data/lib/statsd.rb

@@ -0,0 +1,206 @@
+require "rubygems"
+require "eventmachine"
+require "logger"
+require "socket"
+require "timeout"
+require "uri"
+
+# Hack because the latest amqp gem uses String#bytesize, and not everyone
+# is running ruby 1.8.7.
+if !String.instance_methods.include?(:bytesize)
+  class String
+    alias :bytesize :length
+  end
+end
+
+module StatsD
+  @@timers = Hash.new { |h, k| h[k] = Array.new }
+  @@counters = Hash.new { |h, k| h[k] = 0 }
+  @@logger = Logger.new(STDERR)
+  @@logger.progname = File.basename($0)
+  @@flush_interval = 10
+  @@pct_threshold = 90
+  @@output_func = :output_stdout
+  @@key_suffix = nil
+
+  def self.logger; return @@logger; end
+  def self.logger_output=(output)
+    @@logger = Logger.new(output)
+    @@logger.progname = File.basename($0)
+  end
+
+  def self.flush_interval=(val)
+    @@flush_interval = val.to_i
+  end
+
+  def self.pct_threshold=(val)
+    @@pct_threshold = val.to_i
+  end
+
+  def self.key_suffix=(val)
+    @@key_suffix = val
+  end
+
+  def self.output_url=(url)
+    @@output_url = URI.parse(url)
+    scheme_mapper = {"tcp" => [nil, :output_tcp],
+                     "amqp" => [:setup_amqp, :output_amqp],
+                     "stdout" => [nil, :output_stdout],
+                     }
+    if ! scheme_mapper.has_key?(@@output_url.scheme)
+      raise TypeError, "unsupported scheme in #{url}"
+    end
+
+    setup_func, @@output_func = scheme_mapper[@@output_url.scheme]
+    self.send(setup_func) if setup_func
+  end
+
+  # TODO: option for persistent tcp connection
+  #def setup_tcp
+  #end
+
+  def self.output_tcp(packet)
+    server = TCPSocket.new(@@output_url.host, @@output_url.port)
+    server.puts packet
+    server.close
+  end
+
+  def self.setup_amqp
+    begin
+      require "amqp"
+      require "mq"
+    rescue LoadError
+      @@logger.fatal("missing amqp ruby module. try gem install amqp")
+      exit(1)
+    end
+
+    user = @@output_url.user || ""
+    user, vhost = user.split("@", 2)
+    _, mqtype, mqname = @@output_url.path.split("/", 3)
+    amqp_settings = {
+      :host => @@output_url.host,
+      :port => @@output_url.port || 5672,
+      :user => user,
+      :pass => @@output_url.password,
+      :vhost => vhost || "/",
+    }
+
+    @amqp = AMQP.connect(amqp_settings)
+    @mq = MQ.new(@amqp)
+    @target = nil
+    opts = {:durable => true,
+            :auto_delete => false,
+           }
+ 
+    if @@output_url.query
+      @@output_url.query.split("&").each do |param|
+        k, v = param.split("=", 2)
+        opts[:durable] = false if k == "durable" and v == "false"
+        opts[:auto_delete] = true if k == "autodelete" and v == "true"
+      end
+    end
+
+    @@logger.info(opts.inspect)
+
+    case mqtype
+      when "fanout"
+        @target = @mq.fanout(mqname, opts)
+      when "queue"
+        @target = @mq.queue(mqname, opts)
+      when "topic"
+        @target = @mq.topic(mqname, opts)
+      else
+        raise TypeError, "unknown mq output type #{mqname}"
+    end
+  end
+
+  def self.output_amqp(packet)
+    @target.publish(packet)
+  end
+
+  def self.output_stdout(packet)
+    $stdout.write(packet)
+  end
+
+  def receive_data(packet)
+    bits = packet.split(":")
+    key = bits.shift.gsub(/\s+/, "_") \
+                    .gsub(/\//, "-") \
+                    .gsub(/[^a-zA-Z_\-0-9\.]/, "")
+    bits << "1" if bits.length == 0
+    bits.each do |bit|
+      fields = bit.split("|")
+      if fields.length != 2
+        $stderr.puts "invalid update: #{bit}"
+        next
+      end
+
+      if fields[1] == "ms" # timer update
+        @@timers[key] << fields[0].to_f
+      elsif fields[1] == "c" # counter update
+        count, sample_rate = fields[0].split("@", 2)
+        sample_rate ||= 1
+        #puts "count is #{count.to_f} (#{count})"
+        #puts "multiplier is is #{1 / sample_rate.to_f}"
+        @@counters[key] += count.to_f * (1 / sample_rate.to_f)
+      else
+        $stderr.puts "invalid field in update: #{bit}"
+      end
+    end
+  end
+
+  def self.carbon_update_str
+    updates = []
+    now = Time.now.to_i
+
+    @@timers.each do |key, values|
+      next if values.length == 0
+      values.sort!
+      min = values[0]
+      max = values[-1]
+      mean = min
+      maxAtThreshold = min
+      if values.length > 1
+        threshold_index = ((100 - @@pct_threshold) / 100.0) * values.length
+        threshold_count = values.length - threshold_index.round
+        valid_values = values.slice(0, threshold_count)
+        maxAtThreshold = valid_values[-1]
+
+        sum = 0
+        valid_values.each { |v| sum += v }
+        mean = sum / valid_values.length
+      end
+
+      suffix = @@key_suffix ? ".#{@@key_suffix}" : ""
+      updates << "stats.timers.#{key}.mean#{suffix} #{mean} #{now}"
+      updates << "stats.timers.#{key}.upper#{suffix} #{max} #{now}"
+      updates << "stats.timers.#{key}.upper_#{@@pct_threshold}#{suffix} " \
+                 "#{maxAtThreshold} #{now}"
+      updates << "stats.timers.#{key}.lower#{suffix} #{min} #{now}"
+      updates << "stats.timers.#{key}.count#{suffix} #{values.length} #{now}"
+    end
+
+    @@counters.each do |key, value|
+      suffix = @@key_suffix ? ".#{@@key_suffix}" : ""
+      updates << "stats.#{key}#{suffix} #{value / @@flush_interval} #{now}"
+    end
+
+    @@timers.each { |k, v| @@timers[k] = [] }
+    @@counters.each { |k, v| @@counters[k] = 0 }
+
+    return updates.length == 0 ? nil : updates.join("\n") + "\n"
+  end
+
+  def self.flush
+    s = carbon_update_str
+    return unless s
+
+    begin
+      Timeout::timeout(2) { self.send(@@output_func, s) }
+    rescue Timeout::Error
+      @@logger.warn("timed out sending update to #{@@output_url}")
+    rescue
+      @@logger.warn("error sending update to #{@@output_url}: #{$!}")
+    end
+  end
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification 
+name: petef-statsd
+version: !ruby/object:Gem::Version 
+  hash: 13
+  prerelease: 
+  segments: 
+  - 0
+  - 3
+  version: "0.3"
+platform: ruby
+authors: 
+- Pete Fritchman
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-12-02 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: amqp
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+description: collect and aggregate stats, flush to graphite
+email: petef@databits.net
+executables: 
+- statsd
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/statsd.rb
+- bin/statsd
+homepage: https://github.com/fetep/ruby-statsd
+licenses: 
+- Mozilla Public License (1.1)
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: statsd -- stat collector/aggregator
+test_files: []
+