multi-statsd 0.0.1

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+dump.rdb

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+services:
+  - redis-server
+rvm:
+  - 1.9.3

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in multi-statsd.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Kelley Reynolds
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,53 @@
+# MultiStatsd [![Build Status](https://travis-ci.org/bigcartel/multi-statsd.png?branch=master)](https://travis-ci.org/bigcartel/multi-statsd) [![Code Climate](https://codeclimate.com/github/bigcartel/multi-statsd.png)](https://codeclimate.com/github/bigcartel/multi-statsd)
+
+An eventmachine-based statsd server written in ruby with a modular backend system.
+The motivation behind creating Yet Another Statsd Server was that while the existing ones allowed
+you to specify different backends to flush data to, it was always aggregated the same way and flushed on the same interval.
+This library incurs the overhead of multiple copies of your statistics with the tradeoff that you
+can have backend-specific flush intervals and alternate aggregations of your data.
+
+## Installation
+
+    $ gem install multi-statsd
+    
+## Configuration
+
+Configuration is done via a YAML file which is specified on the command line. Example config file:
+
+    # Host and port to listen on
+    host: 127.0.0.1
+    port: 8125
+    verbosity: 1 # 0-4, 0 being the most verbose
+    logfile: /var/log/multi-statsd.log
+    pidfile: /var/run/multi-statsd.pid
+    daemonize: true
+    backends:
+      stdout_every_5:
+        backend: Stdout
+        flush_interval: 5
+      hottie:
+        backend: Hottie
+        host: 127.0.0.1
+        port: 6379
+        flush_interval: 1
+        seconds_to_retain: 60
+
+
+## Usage
+
+Usage is straight forward:
+
+    $ multi-statsd -c /path/to/config
+
+## Todo
+
+1. Create graphite backend
+2. Create relay backend
+ 
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.rspec_opts = %w[--color]
+  t.pattern = 'spec/**/*_spec.rb'
+end
+
+task :default => [:spec]

data/bin/multi-statsd

@@ -0,0 +1,57 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+require 'yaml'
+require 'optparse'
+
+options = {}
+
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: multi-statsd -c CONFIG_FILE"
+
+  opts.separator ""
+  opts.separator "Options:"
+
+  opts.on("-c CONFIG_FILE", "--config-file CONFIG_FILE", "Configuration file") do |config_file|
+    options = YAML.load_file(config_file).merge(options)
+  end
+
+  opts.on("-d", "--debug", "Shortcut for (daemonize: false, verbosity: 0, log: stdout)") do
+    options['daemonize'] = false
+    options['verbosity'] = 0
+    options['logfile'] = nil
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+
+end
+
+parser.parse!
+
+raise OptionParser::MissingArgument, "A config file must be specified" if options.empty?
+raise "At least one backend must be specified" if !options.has_key?('backends') or options['backends'].empty?
+
+require 'multi-statsd'
+
+Signal.trap("INT") { MultiStatsd.stop }
+Signal.trap("TERM") { MultiStatsd.stop }
+
+log = options['logfile'] ? Logger.new(options['logfile']) : Logger.new(STDOUT)
+log.level = options['verbosity'] || 2
+MultiStatsd.logger = log
+
+# Daemonize if requested
+Process.daemon if options['daemonize']
+
+# Write pidfile if requested
+if options['pidfile']
+  begin
+    File.open(options['pidfile'], File::WRONLY | File::APPEND | File::CREAT) { |fp| fp.write Process.pid }
+  rescue
+    log.error "Unable to write pid file: #{$!.to_s}"
+  end
+end
+
+MultiStatsd.start(options).join

data/etc/multi-statsd.yml

@@ -0,0 +1,17 @@
+# Host and port to listen on
+host: 127.0.0.1
+port: 8125
+verbosity: 1 # 0-4, 0 being the most verbose
+# logfile: /var/log/multi-statsd.log
+# pidfile: /var/run/multi-statsd.pid
+# daemonize: false
+backends:
+  stdout_every_3:
+    backend: Stdout
+    flush_interval: 3
+#  hottie:
+#    backend: Hottie
+#    host: 127.0.0.1
+#    port: 6379
+#    flush_interval: 1
+#    seconds_to_retain: 60

data/lib/multi-statsd.rb

@@ -0,0 +1,69 @@
+require 'logger'
+require 'eventmachine'
+require 'em-logger'
+
+require "multi-statsd/version"
+require 'multi-statsd/backends/base'
+require 'multi-statsd/server'
+
+# MultiStatsd namespace
+module MultiStatsd
+  # Assign a logger
+  # @param [Logger] logger
+  def self.logger=(logger)
+    @logger = logger.kind_of?(EM::Logger) ? logger : EM::Logger.new(logger)
+  end
+  
+  # Return the logger
+  # @return [Logger]
+  def self.logger
+    return @logger if defined?(@logger)
+    log = Logger.new(STDOUT)
+    log.level = 2
+    @logger = EM::Logger.new(log)
+  end
+  
+  # Start up the Eventmachine reactor loop in a separate thread give a set of options.
+  #   This is a test
+  #   This is another test
+  # @param [Hash] options Set of options
+  # @return [thread] Thread which is running the eventmachine loop
+  def self.start(options)
+    thread = Thread.new do
+      EM.run do
+        backends = []
+        options['backends'].each_pair do |name, options|
+          backend = options.delete('backend')
+          begin
+            require "multi-statsd/backends/#{backend.downcase}"
+          rescue LoadError
+            raise MultiStatsd::Backend::Error, "Cannot load file multi-statsd/backends/#{backend.downcase}"
+          end
+          if !MultiStatsd::Backend.const_defined?(backend)
+            raise MultiStatsd::Backend::Error, "No such back end: MultiStatsd::Backend::#{backend}"
+          else
+            logger.info "Adding backend #{backend} :: #{name}"
+            backends << MultiStatsd::Backend.const_get(backend).new(name, options)
+          end
+        end
+
+        EM::open_datagram_socket(
+          (options['host'] || '127.0.0.1'),
+          (options['port'] || 8125),
+          MultiStatsd::Server,
+          backends
+        )
+        logger.info "multi-statsd starting up on #{options['host']}:#{options['port']}"
+      end
+    end
+    
+    thread.abort_on_exception = true
+    thread
+  end
+
+  # Stop the Eventmachine reactor loop
+  def self.stop
+    logger.info "multi-statsd shutting down"
+    EM.next_tick { EM.stop }
+  end
+end

data/lib/multi-statsd/backends/base.rb

@@ -0,0 +1,105 @@
+require 'benchmark'
+require 'thread'
+
+module MultiStatsd
+  # Various backends should be defined in this module
+  module Backend
+    # Error class for MultiStatsd-generated errors
+    class Error < StandardError; end
+
+    # @abstract Subclass and override {#flush} to implement a custom Backend class.
+    class Base
+      attr_reader :counters, :timers, :gauges, :name
+      
+      def initialize(name, options = {})
+        @name, @options = name, options
+        @timers, @gauges, @counters = {}, {}, Hash.new(0)
+        @semaphore = Mutex.new
+        
+        @options['flush_interval'] ||= 15
+
+        post_init
+
+        EventMachine::add_periodic_timer(@options['flush_interval']) do
+          EM.defer { flush }
+        end
+      end
+  
+      # @return [Integer] the flush interval
+      def flush_interval
+        @options['flush_interval']
+      end
+      
+      # Override in subclasses to execute code after initialization (eg. database connection setup)
+      def post_init
+      end
+      
+      # Each backend must implement this method to flush its data
+      def flush
+        raise NotImplementedError
+      end
+      
+      # Reset and return the generated data in a mutex to ensure none are lost
+      # @return [Array] An array consisting of [counters, timers, gauges]
+      def reset_stats
+        @semaphore.synchronize do
+          counters = @counters.dup
+          timers = @timers.dup
+          gauges = @gauges.dup
+          @counters.clear
+          @timers.clear
+          [counters, timers, gauges]
+        end
+      end
+  
+      # Record data in statsd format
+      #   Gauges - cpu:0.15|g
+      #   Timers - api:12|ms
+      #   Counters - bytes:123|c
+      #   Counters with sampling - bytes:123|c|@0.1
+      #   Multiple values - api:12|ms:15|ms:8|ms
+      # @param [String] msg string of data in statsd format
+      # @return [true]
+      def write(msg)
+        msg.each_line do |row|
+          # Fetch our key and records
+          key, *records = row.split(":")
+          
+          # Clean up the key formatting
+          key = format_key(key)
+
+          # Iterate through each record and store the data
+          records.each do |record|
+            value, type, sample_rate = record.split('|')
+            next unless value and type and value =~ /^(?:[\d\.-]+)$/
+
+            case type
+            when "ms"
+              @timers[key] ||= []
+              @timers[key].push(value.to_f)
+            when "c"
+              if sample_rate
+                sample_rate = sample_rate.gsub(/[^\d\.]/, '').to_f
+                sample_rate = 1 if sample_rate <= 0
+                @counters[key] += value.to_f * (1.0 / sample_rate)
+              else
+                @counters[key] += value.to_f
+              end
+            when "g"
+              @gauges[key] = value.to_f
+            end
+          end
+        end
+        
+        true
+      end
+      
+      # Format a given key
+      # @param [String] key from the statsd record
+      # @return [String] formatted key
+      def format_key(key)
+        key.gsub(/\s+/, '_').gsub(/\//, '-').gsub(/[^a-zA-Z_\-0-9\.]/, '')
+      end
+    end
+  end
+end

data/lib/multi-statsd/backends/hottie.rb

@@ -0,0 +1,62 @@
+require 'redis'
+
+# Hottie backend
+# Hottie is a redis-based short-term backend to enable real-time visibility into application behavior.
+# It is specifically designed to enable real-time heatmap/histogram visualizations from timer data.
+class MultiStatsd::Backend::Hottie < MultiStatsd::Backend::Base
+  attr_reader :seconds_to_retain, :samples_to_retain
+
+  # Initialize a connection to redis and configure our samples to retain
+  def post_init
+    @db = Redis.new(
+      :host => (@options['host'] || '127.0.0.1'),
+      :port => (@options['port'] || 6379),
+      :database => (@options['database'] || 1)
+    )
+    @seconds_to_retain = @options['seconds_to_retain'] || 60
+    @samples_to_retain = (@seconds_to_retain / @options['flush_interval']).floor
+  end
+  
+  # Flush the data to redis in the format required for Hottie
+  # @return [Float] The number of seconds it took to aggregate/flush the data to redis
+  def flush
+    counters, timers, gauges = reset_stats
+    ts = Time.new.to_i
+    time = ::Benchmark.realtime do
+      @db.pipelined do
+        if !gauges.empty?
+          @db.hmset "gauges:#{ts}", *(gauges.map { |stat, gauge|  [stat, gauge] }.flatten)
+          @db.expire "gauges:#{ts}", @seconds_to_retain + 5 # Retain a few extra seconds to avoid reporting race
+          @db.sadd "gauges", gauges.keys
+        end
+        @db.lpush "gauge_samples", "gauges:#{ts}"
+        @db.ltrim "gauge_samples", 0, @samples_to_retain - 1
+
+        if !counters.empty?
+          @db.hmset "counters:#{ts}", *(counters.map { |stat, counter|[stat, counter / @options['flush_interval']] }.flatten)
+          @db.expire "counters:#{ts}", @seconds_to_retain + 5 # Retain a few extra seconds to avoid reporting race
+          @db.sadd "counters", counters.keys
+        end
+        @db.lpush "counter_samples", "counters:#{ts}"
+        @db.ltrim "counter_samples", 0, @samples_to_retain - 1
+
+        if !timers.empty?
+          timer_hash = Hash.new(0)
+          @db.hmset "timers:#{ts}", *(timers.map { |stat, data|
+            timer_hash.clear
+            data.each { |d| timer_hash[d.round] += 1 }
+            [stat, Marshal.dump(timer_hash)]
+          })
+          @db.expire "timers:#{ts}", @seconds_to_retain + 5 # Retain a few extra seconds to avoid reporting race
+          @db.sadd "timers", timers.keys
+        end
+        @db.lpush "timer_samples", "timers:#{ts}"
+        @db.ltrim "timer_samples", 0, @samples_to_retain - 1
+      end
+    end
+    MultiStatsd.logger.debug "Hottie flushing took #{"%.3f" % (time * 1000)}ms"
+    time
+  rescue Redis::CannotConnectError
+    MultiStatsd.logger.warning "Unable to connect to redis, skipping flush"
+  end
+end

data/lib/multi-statsd/backends/stdout.rb

@@ -0,0 +1,8 @@
+# Example backend which prints out the all stats to stdout
+class MultiStatsd::Backend::Stdout < MultiStatsd::Backend::Base
+  # Prints the name of this backend, the current time, and inspected stats to stdout
+  # @return [nil]
+  def flush
+    $stdout.puts "[#{@name}:#{Time.now.to_i}] #{reset_stats.inspect}"
+  end
+end

data/lib/multi-statsd/server.rb

@@ -0,0 +1,22 @@
+module MultiStatsd
+  # Eventmachine connection which receives UDP data and writes it to various backends
+  class Server < EventMachine::Connection
+    # Initialize the server with one or more backends
+    # @param [MultiStatsd::Backend] backends One or more backends that will receive data
+    def initialize(backends = [], *args)
+      @backends = [backends].flatten
+      super
+    end
+    
+    # Write out statsd data to each registered backend
+    # @param [String] data Data in statsd format
+    # @return [nil]
+    def receive_data(data)
+      @backends.each do |backend|
+        backend.write(data)
+      end
+      
+      nil
+    end    
+  end    
+end

data/lib/multi-statsd/version.rb

@@ -0,0 +1,4 @@
+module MultiStatsd
+  # Gem version
+  VERSION = "0.0.1"
+end

data/multi-statsd.gemspec

@@ -0,0 +1,31 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'multi-statsd/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "multi-statsd"
+  gem.version       = MultiStatsd::VERSION
+  gem.authors       = ["Kelley Reynolds"]
+  gem.email         = ["kelley@bigcartel.com"]
+  gem.description   = %q{Statsd Server with flexible aggregation and back-end support}
+  gem.summary       = %q{Statsd Server with flexible aggregation and back-end support}
+  gem.homepage      = "https://github.com/bigcartel/multi-statsd"
+  gem.rubyforge_project = "multi-statsd"
+
+  gem.add_dependency "eventmachine"
+  gem.add_dependency "em-logger"
+
+  gem.required_ruby_version     = '>= 1.9.3'
+  gem.add_development_dependency "bundler", ">= 1.0.0"
+  gem.add_development_dependency "simplecov"
+  gem.add_development_dependency "rspec", ">= 2.6.0"
+  gem.add_development_dependency "yard", ">= 0.8"
+  gem.add_development_dependency "rake"
+  gem.add_development_dependency "redis"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+end

data/spec/backends/base_spec.rb

@@ -0,0 +1,251 @@
+# encoding: UTF-8
+require File.expand_path('../spec_helper.rb', File.dirname(__FILE__))
+
+module EventMachine
+  def self.add_periodic_timer(interval)
+    # Don't flush anything automatically
+  end
+end
+
+describe MultiStatsd::Backend::Base do
+  let(:backend) { MultiStatsd::Backend::Base.new('base') }
+
+  describe "default initialization" do
+    it "should have a name" do
+      backend.name.should == 'base'
+    end
+
+    it "should have a default flush_interval" do
+      backend.instance_variable_get(:@options)['flush_interval'].should == 15
+    end
+
+    it "should have a hash of counters, timers, and gauges" do
+      backend.counters.should be_kind_of(Hash)
+      backend.timers.should be_kind_of(Hash)
+      backend.gauges.should be_kind_of(Hash)
+    end
+  end
+  
+  describe "custom initialization" do
+    let(:backend) { MultiStatsd::Backend::Base.new('base', 'flush_interval' => 20) }
+
+    it "should have a default flush_interval" do
+      backend.instance_variable_get(:@options)['flush_interval'].should == 20
+    end
+  end
+  
+  describe "flush" do
+    it "should raise an error" do
+      lambda { backend.flush }.should raise_error NotImplementedError
+    end
+  end
+  
+  describe "reset_stats" do
+    let(:timers) { {'api' => [22, 15] } }
+
+    it "should return an array of hashes" do
+      stats = backend.reset_stats
+      stats.should be_kind_of Array
+      stats.each { |stat|
+        stat.should be_kind_of Hash
+      }
+    end
+    
+    describe "counters" do
+      let(:counters) { {'bytes' => 15 } }
+
+      before(:each) {
+        backend.instance_variable_set(:@counters, counters.dup)
+        @stats = backend.reset_stats[0]
+      }
+
+      it "should return the data" do
+        @stats.should == counters
+      end
+      
+      it "should be a duplicated object" do
+        @stats.object_id.should_not == backend.counters.object_id
+      end
+
+      it "should clear the instance variable" do
+        backend.counters.should be_empty
+      end
+    end
+
+    describe "timers" do
+      let(:timers) { {'api' => [9, 5] } }
+
+      before(:each) {
+        backend.instance_variable_set(:@timers, timers.dup)
+        @stats = backend.reset_stats[1]
+      }
+
+      it "should return the data" do
+        @stats.should == timers
+      end
+      
+      it "should be a duplicated object" do
+        @stats.object_id.should_not == backend.timers.object_id
+      end
+
+      it "should clear the instance variable" do
+        backend.timers.should be_empty
+      end
+    end
+
+    describe "gauges" do
+      let(:gauges) { {'cpu' => 0.15 } }
+
+      before(:each) {
+        backend.instance_variable_set(:@gauges, gauges.dup)
+        @stats = backend.reset_stats[2]
+      }
+
+      it "should return the data" do
+        @stats.should == gauges
+      end
+      
+      it "should be a duplicated object" do
+        @stats.object_id.should_not == backend.gauges.object_id
+      end
+
+      it "should not clear the instance variable" do
+        backend.gauges.should_not be_empty
+      end
+    end
+  end
+  
+  describe "write" do
+    describe "crap" do
+      it "should ignore garbage" do
+        backend.write "blah:123|foo"
+        backend.write "blah:snord|ms"
+        backend.counters.should be_empty
+        backend.gauges.should be_empty
+        backend.timers.should be_empty
+      end
+    end
+    
+    describe "timers" do
+      describe "single" do
+        let(:record) { "api:3|ms" }
+        before(:each) { backend.write record }
+
+        it "should record from scratch" do
+          backend.timers.should == {'api' => [3]}
+        end
+
+        it "should append a time" do
+          backend.write "api:4|ms"
+          backend.timers.should == {'api' => [3, 4]}
+        end
+      end
+
+      describe "multiple" do
+        let(:record) { "api:3|ms:4|ms" }
+        before(:each) { backend.write record }
+
+        it "should record both values" do
+          backend.timers.should == {'api' => [3,4]}
+        end
+      end
+    end
+
+    describe "gauges" do
+      describe "single" do
+        let(:record) { "cpu:0.15|g" }
+        before(:each) { backend.write record }
+
+        it "should record from scratch" do
+          backend.gauges.should == {'cpu' => 0.15}
+        end
+
+        it "should update existing gauge" do
+          backend.write "cpu:0.17|g"
+          backend.gauges.should == {'cpu' => 0.17}
+        end
+      end
+
+      describe "multiple" do
+        let(:record) { "cpu:0.15|g:0.17|g" }
+        before(:each) { backend.write record }
+
+        it "should use the last value" do
+          backend.gauges.should == {'cpu' => 0.17}
+        end
+      end
+    end
+    
+    describe "counters" do
+      describe "single without sample rate" do
+        let(:record) { "bytes:15.2|c" }
+        before(:each) { backend.write record }
+
+        it "should record from scratch" do
+          backend.counters.should == {'bytes' => 15.2}
+        end
+
+        it "should add existing record" do
+          backend.write record
+          backend.counters.should == {'bytes' => 30.4}
+        end
+      end
+
+      describe "single with sample rate" do
+        let(:record) { "bytes:15.2|c|@0.5" }
+        before(:each) { backend.write record }
+
+        it "should record from scratch" do
+          backend.counters.should == {'bytes' => 30.4}
+        end
+
+        it "should add existing record" do
+          backend.write record
+          backend.counters.should == {'bytes' => 60.8}
+        end
+      end
+
+      describe "multiple without sample rate" do
+        let(:record) { "bytes:15.5|c:20.2|c" }
+        before(:each) { backend.write record }
+
+        it "should record from scratch" do
+          backend.counters.should == {'bytes' => 35.7}
+        end
+
+        it "should add existing record" do
+          backend.write record
+          backend.counters.should == {'bytes' => 71.4}
+        end
+      end
+
+      describe "multiple with sample rate" do
+        let(:record) { "bytes:7.75|c|@0.5:10.1|c|@0.5" }
+        before(:each) { backend.write record }
+
+        it "should record from scratch" do
+          backend.counters.should == {'bytes' => 35.7}
+        end
+
+        it "should add existing record" do
+          backend.write record
+          backend.counters.should == {'bytes' => 71.4}
+        end
+      end
+    end
+  end
+
+  describe "format_key" do
+    it "should turn spaces into underscores" do
+      backend.format_key("blah foo").should == "blah_foo"
+    end
+
+    it "should turn slashes into hyphens" do
+      backend.format_key("blah/foo").should == "blah-foo"
+    end
+
+    it "should filter out non-alpha and a few selected special chars" do
+      backend.format_key("blahüfoo*!@&#^%}").should == "blahfoo"
+    end
+  end
+end

data/spec/backends/hottie_spec.rb

@@ -0,0 +1,126 @@
+# encoding: UTF-8
+require File.expand_path('../spec_helper.rb', File.dirname(__FILE__))
+require 'multi-statsd/backends/hottie'
+
+module EventMachine
+  def self.add_periodic_timer(interval)
+    # Don't flush anything automatically
+  end
+end
+
+describe MultiStatsd::Backend::Hottie do
+  let(:backend) { MultiStatsd::Backend::Hottie.new('hottie') }
+
+  describe "post init" do
+    it "should have a connection to redis" do
+      backend.instance_variable_get(:@db).should be_kind_of Redis
+    end
+
+    it "should have a default @seconds_to_retain" do
+      backend.seconds_to_retain.should == 60
+    end
+
+    it "should have a default @seconds_to_retain" do
+      backend = MultiStatsd::Backend::Hottie.new('hottie', 'seconds_to_retain' => 80)
+      backend.seconds_to_retain.should == 80
+    end
+
+    it "should have an appropriate set of samples to retain" do
+      backend.samples_to_retain.should == 4
+    end
+
+    it "should have a default @seconds_to_retain" do
+      backend = MultiStatsd::Backend::Hottie.new('hottie', 'seconds_to_retain' => 80, 'flush_interval' => 10)
+      backend.samples_to_retain.should == 8
+    end
+  end
+  
+  describe "flush" do
+    describe "gauges" do
+      before(:all) {
+        @db = backend.instance_variable_get(:@db)
+        @db.flushdb
+        backend.write "cpu:0.15|g"
+        backend.flush
+      }
+      
+      it "should add the name to the set" do
+        @db.sismember("gauges", "cpu").should be_true
+      end
+
+      it "should have a sample" do
+        samples = @db.lrange "gauge_samples", 0, -1
+        samples.size.should == 1
+        @db.hlen(samples.first).should == 1
+        @db.hget(samples.first, 'cpu').should == "0.15"
+      end
+      
+      it "should roll over on flush if there are too many samples" do
+        backend.samples_to_retain.times do |i|
+          @db.lpush "gauge_samples", "gauge:#{i}"
+        end
+        @db.llen("gauge_samples").should == (backend.samples_to_retain + 1)
+        backend.flush
+        @db.llen("gauge_samples").should == backend.samples_to_retain
+      end
+    end
+
+    describe "counters" do
+      before(:all) {
+        @db = backend.instance_variable_get(:@db)
+        @db.flushdb
+        backend.write "bytes:1200|c"
+        backend.flush
+      }
+      
+      it "should add the name to the set" do
+        @db.sismember("counters", "bytes").should be_true
+      end
+
+      it "should have a sample" do
+        samples = @db.lrange "counter_samples", 0, -1
+        samples.size.should == 1
+        @db.hlen(samples.first).should == 1
+        (@db.hget(samples.first, 'bytes')).to_i.should == (1200 / backend.flush_interval)
+      end
+      
+      it "should roll over on flush if there are too many samples" do
+        backend.samples_to_retain.times do |i|
+          @db.lpush "counter_samples", "counter:#{i}"
+        end
+        @db.llen("counter_samples").should == (backend.samples_to_retain + 1)
+        backend.flush
+        @db.llen("counter_samples").should == backend.samples_to_retain
+      end
+    end
+
+    describe "timers" do
+      before(:all) {
+        @db = backend.instance_variable_get(:@db)
+        @db.flushdb
+        backend.write "api:15.5|ms"
+        backend.flush
+      }
+      
+      it "should add the name to the set" do
+        @db.sismember("timers", "api").should be_true
+      end
+
+      it "should have a rounded sample" do
+        samples = @db.lrange "timer_samples", 0, -1
+        samples.size.should == 1
+        @db.hlen(samples.first).should == 1
+        Marshal.load(@db.hget(samples.first, 'api')).should == {16 => 1}
+      end
+      
+      it "should roll over on flush if there are too many samples" do
+        backend.samples_to_retain.times do |i|
+          @db.lpush "timer_samples", "timer:#{i}"
+        end
+        @db.llen("timer_samples").should == (backend.samples_to_retain + 1)
+        backend.flush
+        @db.llen("timer_samples").should == backend.samples_to_retain
+      end
+    end
+  end
+end

data/spec/backends/stdout_spec.rb

@@ -0,0 +1,20 @@
+# encoding: UTF-8
+require File.expand_path('../spec_helper.rb', File.dirname(__FILE__))
+require 'multi-statsd/backends/stdout'
+
+module EventMachine
+  def self.add_periodic_timer(interval)
+    # Don't flush anything automatically
+  end
+end
+
+describe MultiStatsd::Backend::Stdout do
+  let(:backend) { MultiStatsd::Backend::Stdout.new('stdout') }
+
+  describe "flush behavior" do
+    it "should print to stdout" do
+      $stdout.should_receive(:puts)
+      backend.flush
+    end
+  end
+end

data/spec/multi-statsd_spec.rb

@@ -0,0 +1,22 @@
+# encoding: UTF-8
+require File.expand_path('spec_helper.rb', File.dirname(__FILE__))
+require 'multi-statsd/backends/stdout'
+
+describe MultiStatsd do
+  it "should start and stop the reactor" do
+    MultiStatsd.start({'host' => 'localhost', 'port' => 33333, 'backends' => {'stdout' => {'backend' => 'Stdout'}}})
+    sleep 0.2
+    EM.reactor_running?.should be_true
+    MultiStatsd.stop
+    sleep 0.2
+    EM.reactor_running?.should be_false
+  end
+
+  it "should raise an error on an unknown backend" do
+    lambda {
+      MultiStatsd.start({'host' => 'localhost', 'port' => 33333, 'backends' => {'stdout' => {'backend' => 'Broken'}}})
+      sleep 1
+      
+    }.should raise_error(MultiStatsd::Backend::Error)
+  end
+end

data/spec/server_spec.rb

@@ -0,0 +1,41 @@
+# encoding: UTF-8
+require File.expand_path('spec_helper.rb', File.dirname(__FILE__))
+require 'multi-statsd/backends/stdout'
+
+class MockBackend
+  def write(data); end
+end
+
+describe MultiStatsd::Server do
+  def send_message(message, server='localhost', port=33333)
+    UDPSocket.new.send(message, 0, server, port)
+  end
+
+  let(:message) { 'test:1|c' }
+
+  describe "recording data to backends" do
+    it "Should record data to a single backend" do
+      mock_backend = MockBackend.new
+      mock_backend.should_receive(:write).with(message)
+      EM.run do
+        EM::open_datagram_socket 'localhost', 33333, MultiStatsd::Server, mock_backend
+        send_message message
+        EM.next_tick { EM.stop }
+      end
+    end
+
+    it "should record data to multiple backends" do
+      backends = 3.times.map do
+        backend = MockBackend.new
+        backend.should_receive(:write).with(message)
+        backend
+      end
+
+      EM.run do
+        EM::open_datagram_socket 'localhost', 33333, MultiStatsd::Server, backends
+        send_message message
+        EM.next_tick { EM.stop }
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,6 @@
+require 'simplecov'
+SimpleCov.start do
+  add_filter "/spec/"
+end
+require 'rspec'
+require 'multi-statsd'

metadata

@@ -0,0 +1,205 @@
+--- !ruby/object:Gem::Specification
+name: multi-statsd
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Kelley Reynolds
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-02-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: em-logger
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 2.6.0
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Statsd Server with flexible aggregation and back-end support
+email:
+- kelley@bigcartel.com
+executables:
+- multi-statsd
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/multi-statsd
+- etc/multi-statsd.yml
+- lib/multi-statsd.rb
+- lib/multi-statsd/backends/base.rb
+- lib/multi-statsd/backends/hottie.rb
+- lib/multi-statsd/backends/stdout.rb
+- lib/multi-statsd/server.rb
+- lib/multi-statsd/version.rb
+- multi-statsd.gemspec
+- spec/backends/base_spec.rb
+- spec/backends/hottie_spec.rb
+- spec/backends/stdout_spec.rb
+- spec/multi-statsd_spec.rb
+- spec/server_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/bigcartel/multi-statsd
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1318037612798865336
+requirements: []
+rubyforge_project: multi-statsd
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Statsd Server with flexible aggregation and back-end support
+test_files:
+- spec/backends/base_spec.rb
+- spec/backends/hottie_spec.rb
+- spec/backends/stdout_spec.rb
+- spec/multi-statsd_spec.rb
+- spec/server_spec.rb
+- spec/spec_helper.rb
+has_rdoc: