lookout-statsd 0.7.0

data/.gitignore

@@ -0,0 +1,3 @@
+/pkg
+.rvmrc
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,16 @@
+source 'https://rubygems.org'
+
+gem 'rake'
+
+group :test do
+  if RUBY_VERSION > "1.9"
+    gem 'debugger'
+  else
+    gem 'ruby-debug'
+  end
+
+  gem 'rspec'
+  gem 'cucumber'
+end
+
+gemspec

data/README.md

@@ -0,0 +1,126 @@
+# StatsD
+
+A network daemon for aggregating statistics (counters and timers), rolling them up, then sending them to [graphite][graphite].
+
+
+### Installation
+
+    gem install statsd
+
+### Configuration
+
+Create config.yml to your liking.
+
+Example config.yml
+    ---
+    bind: 127.0.0.1
+    port: 8125
+
+    # Flush interval should be your finest retention in seconds
+    flush_interval: 10
+
+    # Graphite
+    graphite_host: localhost
+    graphite_port: 2003
+
+
+
+### Server
+Run the server:
+
+Flush to Graphite (default):
+    statsd -c config.yml
+
+### Client
+In your client code:
+
+    require 'rubygems'
+    require 'statsd'
+    STATSD = Statsd::Client.new('localhost',8125)
+
+    STATSD.increment('some_counter') # basic incrementing
+    STATSD.increment('system.nested_counter', 0.1) # incrementing with sampling (10%)
+
+    STATSD.decrement(:some_other_counter) # basic decrememting using a symbol
+    STATSD.decrement('system.nested_counter', 0.1) # decrementing with sampling (10%)
+
+    STATSD.timing('some_job_time', 20) # reporting job that took 20ms
+    STATSD.timing('some_job_time', 20, 0.05) # reporting job that took 20ms with sampling (5% sampling)
+
+Concepts
+--------
+
+* *buckets*
+  Each stat is in it's own "bucket". They are not predefined anywhere. Buckets can be named anything that will translate to Graphite (periods make folders, etc)
+
+* *values*
+  Each stat will have a value. How it is interpreted depends on modifiers
+
+* *flush*
+  After the flush interval timeout (default 10 seconds), stats are munged and sent over to Graphite.
+
+Counting
+--------
+
+    gorets:1|c
+
+This is a simple counter. Add 1 to the "gorets" bucket. It stays in memory until the flush interval.
+
+
+Timing
+------
+
+    glork:320|ms
+
+The glork took 320ms to complete this time. StatsD figures out 90th percentile, average (mean), lower and upper bounds for the flush interval.
+
+Sampling
+--------
+
+    gorets:1|c|@0.1
+
+Tells StatsD that this counter is being sent sampled ever 1/10th of the time.
+
+
+Guts
+----
+
+* [UDP][udp]
+  Client libraries use UDP to send information to the StatsD daemon.
+
+* [EventMachine][eventmachine]
+* [Graphite][graphite]
+
+
+Graphite
+--------
+
+Graphite uses "schemas" to define the different round robin datasets it houses (analogous to RRAs in rrdtool):
+
+    [stats]
+    priority = 110
+    pattern = ^stats\..*
+    retentions = 10:2160,60:10080,600:262974
+
+That translates to:
+
+* 6 hours of 10 second data (what we consider "near-realtime")
+* 1 week of 1 minute data
+* 5 years of 10 minute data
+
+This has been a good tradeoff so far between size-of-file (round robin databases are fixed size) and data we care about. Each "stats" database is about 3.2 megs with these retentions.
+
+
+Inspiration
+-----------
+[Etsy's][etsy] [blog post][blog post].
+
+StatsD was inspired (heavily) by the project (of the same name) at Flickr. Here's a post where Cal Henderson described it in depth:
+[Counting and timing](http://code.flickr.com/blog/2008/10/27/counting-timing/). Cal re-released the code recently: [Perl StatsD](https://github.com/iamcal/Flickr-StatsD)
+
+
+[graphite]: http://graphite.wikidot.com
+[etsy]: http://www.etsy.com
+[blog post]: http://codeascraft.etsy.com/2011/02/15/measure-anything-measure-everything/
+[udp]: http://enwp.org/udp
+[eventmachine]: http://rubyeventmachine.com/

data/Rakefile

@@ -0,0 +1,10 @@
+require 'bundler'
+require 'rspec/core/rake_task'
+
+Bundler::GemHelper.install_tasks
+
+task :default => [:test]
+
+RSpec::Core::RakeTask.new(:test) do |t|
+  t.rspec_opts = '--order random --fail-fast --color --format d'
+end

data/bin/statsd

@@ -0,0 +1,45 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'yaml'
+require 'optparse'
+
+begin
+  ORIGINAL_ARGV = ARGV.dup
+  options = {}
+
+  parser = OptionParser.new do |opts|
+    opts.banner = "Usage: statsd [options]"
+
+    opts.separator ""
+    opts.separator "Options:"
+
+    opts.on("-cCONFIG", "--config-file CONFIG", "Configuration file") do |x|
+      options[:config] = x
+    end
+
+    opts.on("-h", "--help", "Show this message") do
+      puts opts
+      exit
+    end
+  end
+
+  parser.parse!
+
+  # dispatch
+  if !options[:config]
+    puts parser.help
+  else
+    require 'statsd'
+    require 'statsd/server'
+    Statsd::Server::Daemon.new.run(options)
+  end
+rescue Exception => e
+  if e.instance_of?(SystemExit)
+    raise
+  else
+    puts 'Uncaught exception'
+    puts e.message
+    puts e.backtrace.join("\n")
+  end
+end

data/config.yml

@@ -0,0 +1,10 @@
+---
+bind: 127.0.0.1
+port: 8125
+
+# Flush interval should be your finest retention in seconds
+flush_interval: 10
+
+# Graphite
+graphite_host: localhost
+graphite_port: 2003

data/lib/statsd.rb

@@ -0,0 +1,130 @@
+require 'socket'
+require 'resolv'
+
+module Statsd
+  # initialize singleton instance in an initializer
+  def self.create_instance(opts={})
+    raise "Already initialized Statsd" if defined? @@instance
+    @@instance ||= Client.new(opts)
+  end
+
+  # access singleton instance, which must have been initialized with #create_instance
+  def self.instance
+    raise "Statsd has not been initialized" unless @@instance
+    @@instance
+  end
+
+  class Client
+    attr_accessor :host, :port, :prefix
+
+    def initialize(opts={})
+      @host = opts[:host] || 'localhost'
+      @port = opts[:port] || 8125
+      @prefix = opts[:prefix]
+    end
+
+    def host_ip_addr
+      @host_ip_addr ||= Resolv.getaddress(host)
+    end
+
+    def host=(h)
+      @host_ip_addr = nil
+      @host = h
+    end
+
+    # +stat+ to log timing for
+    # +time+ is the time to log in ms
+    def timing(stat, time = nil, sample_rate = 1)
+      value = nil
+      if block_given?
+        start_time = Time.now.to_f
+        value = yield
+        time = ((Time.now.to_f - start_time) * 1000).floor
+      end
+
+      if @prefix
+        stat = "#{@prefix}.#{stat}"
+      end
+
+      send_stats("#{stat}:#{time}|ms", sample_rate)
+      value
+    end
+
+    # +stats+ can be a string or an array of strings
+    def increment(stats, sample_rate = 1)
+      if @prefix
+        stats = "#{@prefix}.#{stats}"
+      end
+      update_counter stats, 1, sample_rate
+    end
+
+    # +stats+ can be a string or an array of strings
+    def decrement(stats, sample_rate = 1)
+      if @prefix
+        stats = "#{@prefix}.#{stats}"
+      end
+      update_counter stats, -1, sample_rate
+    end
+
+    # +stats+ can be a string or array of strings
+    def update_counter(stats, delta = 1, sample_rate = 1)
+      stats = Array(stats)
+      send_stats(stats.map { |s| "#{s}:#{delta}|c" }, sample_rate)
+    end
+
+    # +stats+ is a hash
+    def gauge(stats)
+      send_stats(stats.map { |s,val|
+                   if @prefix
+                     s = "#{@prefix}.#{s}"
+                   end
+                   "#{s}:#{val}|g"
+                 })
+    end
+
+    private
+
+    def send_stats(data, sample_rate = 1)
+      data = Array(data)
+      sampled_data = []
+
+      # Apply sample rate if less than one
+      if sample_rate < 1
+        data.each do |d|
+          if rand <= sample_rate
+            sampled_data << "#{d}@#{sample_rate}"
+          end
+        end
+        data = sampled_data
+      end
+
+      return if data.empty?
+
+      raise "host and port must be set" unless host && port
+
+      begin
+        sock = UDPSocket.new
+        data.each do |d|
+          sock.send(d, 0, host, port)
+        end
+      rescue # silent but deadly
+      ensure
+        sock.close
+      end
+      true
+    end
+
+  end
+
+  module Rails
+    # to monitor all actions for this controller (and its descendents) with graphite,
+    # use "around_filter Statsd::Rails::ActionTimerFilter"
+    class ActionTimerFilter
+      def self.filter(controller, &block)
+        key = "requests.#{controller.controller_name}.#{controller.params[:action]}"
+        Statsd.instance.timing(key, &block)
+      end
+    end
+  end
+
+end

data/lib/statsd/echos.rb

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+#
+
+require 'rubygems'
+require 'eventmachine'
+
+module EchoServer
+  def post_init
+    puts "-- someone connected to the server!"
+  end
+
+  def receive_data data
+    puts data
+    send_data ">>> you sent: #{data}"
+  end
+end
+
+EventMachine::run {
+  EventMachine::start_server "127.0.0.1", 2003, EchoServer
+  puts 'running dummy graphite echo server on 2003'
+}

data/lib/statsd/graphite.rb

@@ -0,0 +1,70 @@
+require 'benchmark'
+require 'eventmachine'
+
+
+module Statsd
+  class Graphite < EM::Connection
+    attr_accessor :counters, :timers, :flush_interval
+
+    def flush_stats
+      puts "#{Time.now} Flushing #{counters.count} counters and #{timers.count} timers to Graphite."
+
+      stat_string = ''
+
+      ts = Time.now.to_i
+      num_stats = 0
+
+      # store counters
+      counters.each_pair do |key,value|
+        message = "#{key} #{value} #{ts}\n"
+        stat_string += message
+        counters[key] = 0
+
+        num_stats += 1
+      end
+
+      # store timers
+      timers.each_pair do |key, values|
+        if (values.length > 0)
+          pct_threshold = 90
+          values.sort!
+          count = values.count
+          min = values.first
+          max = values.last
+
+          mean = min
+          max_at_threshold = max
+
+          if (count > 1)
+            # average all the timing data
+            sum = values.inject( 0 ) { |s,x| s+x }
+            mean = sum / values.count
+
+            # strip off the top 100-threshold
+            threshold_index = (((100 - pct_threshold) / 100.0) * count).round
+            values = values[0..-threshold_index]
+            max_at_threshold = values.last
+          end
+
+          message = ""
+          message += "#{key}.mean #{mean} #{ts}\n"
+          message += "#{key}.upper #{max} #{ts}\n"
+          message += "#{key}.upper_#{pct_threshold} #{max_at_threshold} #{ts}\n"
+          message += "#{key}.lower #{min} #{ts}\n"
+          message += "#{key}.count #{count} #{ts}\n"
+          stat_string += message
+
+          timers[key] = []
+
+          num_stats += 1
+        end
+      end
+
+      stat_string += "statsd.numStats #{num_stats} #{ts}\n"
+
+      # send to graphite
+      send_data stat_string
+      close_connection_after_writing
+    end
+  end
+end

data/lib/statsd/server.rb

@@ -0,0 +1,82 @@
+require 'eventmachine'
+require 'yaml'
+require 'erb'
+
+require 'statsd/graphite'
+
+module Statsd
+  module Server
+    Version = '0.5.5'
+
+    FLUSH_INTERVAL = 10
+    COUNTERS = {}
+    TIMERS = {}
+    GAUGES = {}
+
+    def post_init
+      puts "statsd server started!"
+    end
+
+    def self.get_and_clear_stats!
+      counters = COUNTERS.dup
+      timers = TIMERS.dup
+      gauges = GAUGES.dup
+      COUNTERS.clear
+      TIMERS.clear
+      GAUGES.clear
+      [counters,timers,gauges]
+    end
+
+    def receive_data(msg)
+      msg.split("\n").each do |row|
+        bits = row.split(':')
+        key = bits.shift.gsub(/\s+/, '_').gsub(/\//, '-').gsub(/[^a-zA-Z_\-0-9\.]/, '')
+        bits.each do |record|
+          sample_rate = 1
+          fields = record.split("|")
+          if fields.nil? || fields.count < 2
+            next
+          end
+          if (fields[1].strip == "ms")
+            TIMERS[key] ||= []
+            TIMERS[key].push(fields[0].to_i)
+          elsif (fields[1].strip == "c")
+            if (fields[2] && fields[2].match(/^@([\d\.]+)/))
+              sample_rate = fields[2].match(/^@([\d\.]+)/)[1]
+            end
+            COUNTERS[key] ||= 0
+            COUNTERS[key] += (fields[0].to_i || 1) * (1.0 / sample_rate.to_f)
+          elsif (fields[1].strip == "g")
+            GAUGES[key] ||= (fields[0].to_i || 0)
+          else
+            puts "Invalid statistic #{fields.inspect} received; ignoring"
+          end
+        end
+      end
+    end
+
+    class Daemon
+      def run(options)
+        config = YAML::load(ERB.new(IO.read(options[:config])).result)
+
+        EventMachine::run do
+          EventMachine::open_datagram_socket(config['bind'], config['port'], Statsd::Server)
+          puts "Listening on #{config['bind']}:#{config['port']}"
+
+          # Periodically Flush
+          EventMachine::add_periodic_timer(config['flush_interval']) do
+            counters,timers = Statsd::Server.get_and_clear_stats!
+
+            EventMachine.connect config['graphite_host'], config['graphite_port'], Statsd::Graphite do |conn|
+              conn.counters = counters
+              conn.timers = timers
+              conn.flush_interval = config['flush_interval']
+              conn.flush_stats
+            end
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/statsd/test.rb

@@ -0,0 +1,3 @@
+require './graphite'
+counters = timers = []
+#Statsd::Graphite.flush_stats(counters,timers)

data/netcat-example.sh

@@ -0,0 +1,5 @@
+nc -w 1 -u 127.0.0.1 8125 << EOF
+globs:1|c
+gorets:1|c|@0.1
+glork:320|ms
+EOF

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+$:.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
+require 'statsd'
+require 'statsd/server'

data/spec/statsd/server_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+
+describe Statsd::Server do
+  include Statsd::Server
+
+  describe :receive_data do
+    it 'should not vomit on bad data' do
+      bad_data = "dev.rwygand.app.flexd.exception.no action responded to index. actions: authenticate, authentication_request, authorization, bubble_stacktrace?, decode_credentials, encode_credentials, not_found, and user_name_and_password:1|c"
+
+      expect {
+        receive_data(bad_data)
+      }.not_to raise_error
+    end
+  end
+end

data/spec/statsd_spec.rb

@@ -0,0 +1,130 @@
+require 'spec_helper'
+
+describe Statsd do
+  describe '#create_instance' do
+    after(:each) do
+      Statsd.send(:remove_class_variable, :@@instance)
+    end
+
+    it 'should create an instance' do
+      Statsd.create_instance
+      Statsd.instance.should_not be nil
+    end
+
+    it 'should raise if called twice' do
+      Statsd.create_instance
+      expect { Statsd.create_instance }.to raise_error
+    end
+  end
+
+  describe '#instance' do
+    it 'should raise if not created' do
+      expect { Statsd.instance }.to raise_error
+    end
+  end
+end
+
+describe Statsd::Client do
+  describe '#initialize' do
+    it 'should work without arguments' do
+      c = Statsd::Client.new
+      c.should_not be nil
+    end
+
+    it 'should accept a :host keyword argument' do
+      host = 'zombo.com'
+      c = Statsd::Client.new(:host => host)
+      c.host.should match(host)
+    end
+
+    it 'should accept a :port keyword argument' do
+      port = 1337
+      c = Statsd::Client.new(:port => port)
+      c.port.should == port
+    end
+
+    it 'should accept a :prefix keyword argument' do
+      prefix = 'dev'
+      c = Statsd::Client.new(:prefix => prefix)
+      c.prefix.should match(prefix)
+    end
+  end
+
+  describe '#timing' do
+    let(:c) { Statsd::Client.new }
+
+    it 'should pass the sample rate along' do
+      sample = 10
+      c.should_receive(:send_stats).with(anything(), sample)
+      c.timing('foo', 1, sample)
+    end
+
+    it 'should use the right stat name' do
+      c.should_receive(:send_stats).with('foo:1|ms', anything())
+      c.timing('foo', 1)
+    end
+
+    it 'should prefix its stats if it has a prefix' do
+      c.should_receive(:send_stats).with('dev.foo:1|ms', anything())
+      c.prefix = 'dev'
+      c.timing('foo', 1)
+    end
+
+    it 'should wrap a block correctly' do
+      # Pretend our block took one second
+      c.should_receive(:send_stats).with('foo:1000|ms', anything())
+      Time.stub_chain(:now, :to_f).and_return(1, 2)
+
+      c.timing('foo') do
+        true.should be true
+      end
+    end
+
+    it 'should return the return value from the block' do
+      # Pretend our block took one second
+      c.should_receive(:send_stats).with('foo:1000|ms', anything())
+      Time.stub_chain(:now, :to_f).and_return(1, 2)
+
+      value = c.timing('foo') { 1337 }
+      value.should == 1337
+    end
+  end
+
+  describe '#increment' do
+    let(:c) { Statsd::Client.new }
+
+    it 'should prepend the prefix if it has one' do
+      c.prefix = 'dev'
+      c.should_receive(:update_counter).with('dev.foo', anything(), anything())
+      c.increment('foo')
+    end
+  end
+
+  describe '#decrement' do
+    let(:c) { Statsd::Client.new }
+
+    it 'should prepend the prefix if it has one' do
+      c.prefix = 'dev'
+      c.should_receive(:update_counter).with('dev.foo', anything(), anything())
+      c.decrement('foo')
+    end
+  end
+
+  describe '#gauge' do
+    let(:c) { Statsd::Client.new }
+
+    it 'should encode the values correctly' do
+      c.should_receive(:send_stats).with do |array|
+        array.should include('foo:1|g')
+        array.should include('bar:2|g')
+      end
+      c.gauge('foo' => 1, 'bar' => 2)
+    end
+
+    it 'should prepend the prefix if it has one' do
+      c.prefix = 'dev'
+      c.should_receive(:send_stats).with(['dev.foo:1|g'])
+      c.gauge('foo' => 1)
+    end
+  end
+end

data/stats.rb

@@ -0,0 +1,28 @@
+require 'eventmachine'
+require 'statsd'
+require 'statsd/server'
+require 'statsd/graphite'
+
+require 'yaml'
+require 'erb'
+
+ROOT = File.expand_path(File.dirname(__FILE__))
+APP_CONFIG = YAML::load(ERB.new(IO.read(File.join(ROOT,'config.yml'))).result)
+
+# Start the server
+EventMachine::run do
+  EventMachine::open_datagram_socket('127.0.0.1', 8125, Statsd::Server)
+  EventMachine::add_periodic_timer(APP_CONFIG['flush_interval']) do
+     counters,timers = Statsd::Server.get_and_clear_stats!
+
+     # Graphite
+     EventMachine.connect APP_CONFIG['graphite_host'], APP_CONFIG['graphite_port'], Statsd::Graphite do |conn|
+       conn.counters = counters
+       conn.timers = timers
+       conn.flush_interval = 10
+       conn.flush_stats
+     end
+  end
+
+
+end

data/statsd.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name        = "lookout-statsd"
+  s.version     = "0.7.#{ENV['BUILD_NUMBER'] || 'dev'}"
+  s.platform    = Gem::Platform::RUBY
+
+  s.authors     = ['R. Tyler Croy', 'Andrew Coldham', 'Ben VandenBos']
+  s.email       = ['rtyler.croy@mylookout.com']
+  s.homepage    = "https://github.com/lookout/statsd"
+
+  s.summary     = "Ruby version of statsd."
+  s.description = "A network daemon for aggregating statistics (counters and timers), rolling them up, then sending them to graphite."
+
+  s.required_rubygems_version = ">= 1.3.6"
+
+  s.add_dependency "eventmachine",  ">= 0.12.10"
+  s.add_dependency "erubis",        ">= 2.6.6"
+
+  s.files        = `git ls-files`.split("\n")
+  s.executables  = `git ls-files`.split("\n").map{|f| f =~ /^bin\/(.*)/ ? $1 : nil}.compact
+  s.require_path = 'lib'
+end
+

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification 
+name: lookout-statsd
+version: !ruby/object:Gem::Version 
+  hash: 3
+  prerelease: 
+  segments: 
+  - 0
+  - 7
+  - 0
+  version: 0.7.0
+platform: ruby
+authors: 
+- R. Tyler Croy
+- Andrew Coldham
+- Ben VandenBos
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2013-04-08 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 59
+        segments: 
+        - 0
+        - 12
+        - 10
+        version: 0.12.10
+  prerelease: false
+  type: :runtime
+  requirement: *id001
+  name: eventmachine
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 27
+        segments: 
+        - 2
+        - 6
+        - 6
+        version: 2.6.6
+  prerelease: false
+  type: :runtime
+  requirement: *id002
+  name: erubis
+description: A network daemon for aggregating statistics (counters and timers), rolling them up, then sending them to graphite.
+email: 
+- rtyler.croy@mylookout.com
+executables: 
+- statsd
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- bin/statsd
+- config.yml
+- lib/statsd.rb
+- lib/statsd/echos.rb
+- lib/statsd/graphite.rb
+- lib/statsd/server.rb
+- lib/statsd/test.rb
+- netcat-example.sh
+- spec/spec_helper.rb
+- spec/statsd/server_spec.rb
+- spec/statsd_spec.rb
+- stats.rb
+- statsd.gemspec
+homepage: https://github.com/lookout/statsd
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 23
+      segments: 
+      - 1
+      - 3
+      - 6
+      version: 1.3.6
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Ruby version of statsd.
+test_files: []
+