mouth 0.8.0

data/lib/mouth/source.rb

@@ -0,0 +1,76 @@
+module Mouth
+  class Source
+    
+    attr_accessor :all_attributes
+    
+    def initialize(tuple)
+      self.all_attributes = tuple
+    end
+    
+    # Returns an array of tuples:
+    # [{source: "auth.inline_logged_in", kind: "counter|timer"}, ...]
+    def self.all
+      col_names = Mouth.mongo.collections.collect(&:name) - %w(dashboards graphs system.indexes)
+      col_names.select! {|c| c =~ /^mouth_.+/ }
+      
+      tuples = []
+      col_names.each do |col_name|
+        namespace = col_name.match(/mouth_(.+)/)[1]
+        counters, timers = all_for_collection(Mouth.collection(col_name))
+        counters.each {|s| tuples << {:source => "#{namespace}.#{s}", :kind => "counter"} }
+        timers.each {|s| tuples << {:source => "#{namespace}.#{s}", :kind => "timer"} }
+      end
+      
+      tuples.sort_by {|t| "#{t[:kind]}#{t[:source]}" }.collect {|t| new(t) }
+    end
+    
+    def self.all_for_collection(col, window = Time.now.to_i / 60 - 120)
+      map_function = <<-JS
+        function() {
+          var k, vh;
+          for (k in this.c) {
+            vh = {};
+            vh[k] = true;
+            emit("counters", {ks: vh});
+          }
+          for (k in this.m) {
+            vh = {};
+            vh[k] = true;
+            emit("timers", {ks: vh});
+          }
+        }
+      JS
+      
+      reduce_function = <<-JS
+        function(key, values) {
+          var ret = {ks: {}}, k;
+          
+          values.forEach(function(value) {
+            for (k in value.ks) {
+              ret.ks[k] = true
+            }
+          });
+          return ret;
+        }
+      JS
+      
+      result = col.map_reduce(map_function, reduce_function, :out => {:inline => true}, :raw => true, :query => {"t" => {"$gte" => window}})
+      
+      counters = []
+      timers = []
+      if result && result["results"].is_a?(Array)
+        result["results"].each do |r|
+          k, v = r["_id"], r["value"]
+          if k == "timers"
+            timers << v["ks"].keys
+          elsif k == "counters"
+            counters << v["ks"].keys
+          end
+        end
+      end
+      
+      [counters.flatten.compact.uniq, timers.flatten.compact.uniq]
+    end
+    
+  end
+end

data/lib/mouth/sucker.rb

@@ -0,0 +1,235 @@
+require 'em-mongo'
+require 'eventmachine'
+
+module Mouth
+  
+  class SuckerConnection < EM::Connection
+    attr_accessor :sucker
+  
+    def receive_data(data)
+      Mouth.logger.debug "UDP packet: '#{data}'"
+  
+      sucker.store!(data)
+    end
+  end
+  
+  class Sucker
+    
+    # Host/Port to suck UDP packets on
+    attr_accessor :host
+    attr_accessor :port
+
+    # Actual EM::Mongo connection
+    attr_accessor :mongo_db
+    
+    # Info to connect to mongo
+    attr_accessor :mongo_db_name
+    attr_accessor :mongo_hosts
+    
+    # Accumulators of our data
+    attr_accessor :counters
+    attr_accessor :timers
+    
+    # Stats
+    attr_accessor :udp_packets_received
+    attr_accessor :mongo_flushes
+    
+    def initialize(options = {})
+      self.host = options[:host] || "localhost"
+      self.port = options[:port] || 8889
+      self.mongo_db_name = options[:mongo_db_name] || "mouth"
+      self.mongo_hosts = options[:mongo_hosts] || ["localhost"]
+      
+      self.udp_packets_received = 0
+      self.mongo_flushes = 0
+      
+      self.counters = {}
+      self.timers = {}
+    end
+    
+    def suck!
+      EM.run do
+        # Connect to mongo now
+        self.mongo_db
+
+        EM.open_datagram_socket host, port, SuckerConnection do |conn|
+          conn.sucker = self
+        end
+        
+        EM.add_periodic_timer(10) do
+          Mouth.logger.info "Counters: #{self.counters.inspect}"
+          Mouth.logger.info "Timers: #{self.timers.inspect}"
+          self.flush!
+          self.set_procline!
+        end
+
+        EM.next_tick do
+          Mouth.logger.info "Mouth reactor started..."
+          self.set_procline!
+        end
+      end
+    end
+    
+    # counter: gorets:1|c
+    # counter w/ sampling: gorets:1|c|@0.1
+    # timer: glork:320|ms
+    # (future) gauge: gaugor:333|g
+    def store!(data)
+      key_value, command_sampling = data.split("|", 2)
+      key, value = key_value.to_s.split(":")
+      command, sampling = command_sampling.split("|")
+      
+      key = Mouth.parse_key(key).join(".")
+      value = value.to_f
+      
+      ts = minute_timestamps
+      
+      if command == "ms"
+        self.timers[ts] ||= {}
+        self.timers[ts][key] ||= []
+        self.timers[ts][key] << value
+      elsif command == "c"
+        factor = 1.0
+        if sampling
+          factor = sampling.sub("@", "").to_f
+          factor = (factor == 0.0 || factor > 1.0) ? 1.0 : 1.0 / factor
+        end
+        self.counters[ts] ||= {}
+        self.counters[ts][key] ||= 0.0
+        self.counters[ts][key] += value * factor
+      end
+      
+      self.udp_packets_received += 1
+    end
+    
+    def flush!
+      ts = minute_timestamps
+      limit_ts = ts - 1
+      mongo_docs = {}
+      
+      # We're going to construct mongo_docs which look like this:
+      # "mycollections:234234": {  # NOTE: this timpstamp will be popped into .t = 234234
+      #   c: {
+      #     happenings: 37,
+      #     affairs: 3
+      #   },
+      #   m: {
+      #     occasions: {...}
+      #   }
+      # }
+      
+      self.counters.each do |cur_ts, counters_to_save|
+        if cur_ts <= limit_ts
+          counters_to_save.each do |counter_key, value|
+            ns, sub_key = Mouth.parse_key(counter_key)
+            mongo_key = "#{ns}:#{ts}"
+            mongo_docs[mongo_key] ||= {}
+            
+            cur_mongo_doc = mongo_docs[mongo_key]
+            cur_mongo_doc["c"] ||= {}
+            cur_mongo_doc["c"][sub_key] = value
+          end
+          
+          self.counters.delete(cur_ts)
+        end
+      end
+      
+      self.timers.each do |cur_ts, timers_to_save|
+        if cur_ts <= limit_ts
+          timers_to_save.each do |timer_key, values|
+            ns, sub_key = Mouth.parse_key(timer_key)
+            mongo_key = "#{ns}:#{ts}"
+            mongo_docs[mongo_key] ||= {}
+            
+            cur_mongo_doc = mongo_docs[mongo_key]
+            cur_mongo_doc["m"] ||= {}
+            cur_mongo_doc["m"][sub_key] = analyze_timer(values)
+          end
+          
+          self.timers.delete(cur_ts)
+        end
+      end
+      
+      save_documents!(mongo_docs)
+    end
+    
+    def save_documents!(mongo_docs)
+      Mouth.logger.info "Saving Docs: #{mongo_docs.inspect}"
+      
+      mongo_docs.each do |key, doc|
+        ns, ts = key.split(":")
+        collection_name = "mouth_#{ns}"
+        doc["t"] = ts.to_i
+        
+        self.mongo_db.collection(collection_name).insert(doc)
+      end
+      
+      self.mongo_flushes += 1 if mongo_docs.any?
+    end
+    
+    def mongo_db
+      @mongo_db ||= begin
+        if self.mongo_hosts.length == 1
+          EM::Mongo::Connection.new(self.mongo_hosts.first).db(self.mongo_db_name)
+        else
+          raise "TODO: ability to connect to a replica set."
+        end
+      end
+    end
+    
+    def set_procline!
+      $0 = "mouth [started] [UDP Recv: #{self.udp_packets_received}] [Mongo saves: #{self.mongo_flushes}]"
+    end
+    
+    private
+    
+    def minute_timestamps
+      Time.now.to_i / 60
+    end
+    
+    def analyze_timer(values)
+      values.sort!
+      
+      count = values.length
+      min = values[0]
+      max = values[-1]
+      mean = nil
+      sum = 0.0
+      median = median_for(values)
+      stddev = 0.0
+      
+      values.each {|v| sum += v }
+      mean = sum / count
+      
+      values.each do |v|
+        devi = v - mean
+        stddev += (devi * devi)
+      end
+      
+      stddev = Math.sqrt(stddev / count)
+      
+      {
+        "count" => count,
+        "min" => min,
+        "max" => max,
+        "mean" => mean,
+        "sum" => sum,
+        "median" => median,
+        "stddev" => stddev,
+      }
+    end
+    
+    def median_for(values)
+      count = values.length
+      middle = count / 2
+      if count == 0
+        return 0
+      elsif count % 2 == 0
+        return (values[middle] + values[middle - 1]).to_f / 2
+      else
+        return values[middle]
+      end
+    end
+        
+  end # class Sucker
+end # module

data/lib/mouth/version.rb

@@ -0,0 +1,3 @@
+module Mouth
+  VERSION = "0.8.0"
+end

data/mouth.gemspec

@@ -0,0 +1,28 @@
+require './lib/mouth/version'
+
+Gem::Specification.new do |s|
+  s.name = 'mouth'
+  s.version = Mouth::VERSION
+  s.author = 'Jonathan Novak'
+  s.email = 'jnovak@gmail.com'
+  s.homepage = 'http://github.com/cypriss/mouth'
+  s.summary = 'Collect and view stats in real time.'
+  s.description = 'Ruby Daemon collects statistics via UDP packets, stores them in Mongo, and then views them via a user friendly Sintra app.'
+
+  s.files = `git ls-files`.split("\n")
+  s.test_files = `git ls-files test`.split("\n")
+  s.require_path = 'lib'
+  s.bindir = 'bin'
+  s.executables << 'mouth' << 'mouth-endoscope'
+
+  s.add_runtime_dependency 'em-mongo', '~> 0.4.2'       # For the EM collector
+  s.add_runtime_dependency 'mongo', '~> 1.6'            # For the sinatra app
+  s.add_runtime_dependency 'bson_ext', '~> 1.6'
+  s.add_runtime_dependency 'eventmachine', '~> 0.12.10'
+  s.add_runtime_dependency 'vegas', '~> 0.1.8'
+  s.add_runtime_dependency 'sinatra', '~> 1.3.1'
+  s.add_runtime_dependency 'yajl-ruby', '~> 1.1.0'
+  
+  s.required_ruby_version = '>= 1.9.2'
+  s.required_rubygems_version = '>= 1.3.4'
+end

data/test/sequence_test.rb

@@ -0,0 +1,163 @@
+$LOAD_PATH.unshift 'test'
+require 'test_helper'
+
+require 'mouth/sequence'
+
+class SequenceTest < Test::Unit::TestCase
+  def setup
+    @start_time = Time.new(2012, 4, 1, 9, 30, 0, "-07:00")
+    @end_time = Time.new(2012, 4, 1, 11, 30, 0, "-07:00")
+    @namespace, @metric = Mouth.parse_key("test.test")
+    
+    @start_timestamp = @start_time.to_i / 60
+    @end_timestamp = @end_time.to_i / 60
+    
+    counter = 1
+    timer = {"count" => 5, "min" => 1, "max" => 10, "mean" => 5, "sum" => 20, "median" => 4, "stddev" => 2.3}
+    (@start_timestamp..@end_timestamp).each do |t|
+      # Insert the document into mongo
+      Mouth.collection(Mouth.mongo_collection_name(@namespace)).update({"t" => t}, {"$set" => {"c.#{@metric}" => counter, "m.#{@metric}" => timer}}, :upsert => true)
+    end
+  end
+  
+  def test_minute_counter_sequences
+    seq = Mouth::Sequence.new("#{@namespace}.#{@metric}", :start_time => @start_time, :end_time => @end_time)
+
+    sequences = seq.sequences
+    
+    assert_equal ["test"], sequences.keys
+    assert_equal 1, sequences.values.length
+    
+    values = sequences.values.first
+    
+    assert_equal 121, values.length
+    
+    assert values.all? {|v| v == 1} # The 'test' sequence is all ones
+  end
+  
+  def test_15_minute_counter_sequences
+    seq = Mouth::Sequence.new("#{@namespace}.#{@metric}", :granularity_in_minutes => 15, :start_time => @start_time, :end_time => @end_time)
+    
+    sequences = seq.sequences
+    
+    assert_equal ["test"], sequences.keys
+    assert_equal 1, sequences.values.length
+    
+    values = sequences.values.first
+    
+    assert_equal 9, values.length
+    assert_equal [15.0, 15.0, 15.0, 15.0, 15.0, 15.0, 15.0, 15.0, 1.0], values
+  end
+  
+  def test_15_minute_timer_sequences_basic
+    seq = Mouth::Sequence.new("#{@namespace}.#{@metric}", :kind => :timer, :granularity_in_minutes => 15, :start_time => @start_time, :end_time => @end_time)
+    
+    sequences = seq.sequences
+    assert_equal ["test"], sequences.keys
+    assert_equal 1, sequences.values.length
+    
+    values = sequences.values.first
+    
+    assert_equal 9, values.length
+    
+    timer =  {"count"=>75,
+              "min"=>1,
+              "max"=>10,
+              "mean"=>4,
+              "sum"=>300,
+              "median"=>4,
+              "stddev"=>3.780211634287159}
+    assert_equal timer, values.first
+  end
+  
+  def test_15_minute_timer_sequences_one_grouping
+    @start_time = Time.new(2012, 4, 1, 9, 30, 0, "-07:00")
+    @end_time = Time.new(2012, 4, 1, 9, 44, 0, "-07:00")
+    
+    @start_timestamp = @start_time.to_i / 60
+    @end_timestamp = @end_time.to_i / 60
+    
+    col = Mouth.collection(Mouth.mongo_collection_name("test"))
+    col.remove
+    
+    # NOTE: this data is totally fake, maybe we can get some real data
+    timers = [
+      {"count"=>3, "min"=>15, "max"=>30, "mean"=>100, "sum"=>300, "median"=>4, "stddev"=>1},
+      {"count"=>6, "min"=>14, "max"=>32, "mean"=>90, "sum"=>540, "median"=>5, "stddev"=>2},
+      {"count"=>9, "min"=>13, "max"=>30, "mean"=>85, "sum"=>765, "median"=>6, "stddev"=>3},
+      {"count"=>12, "min"=>12, "max"=>30, "mean"=>80, "sum"=>960, "median"=>7, "stddev"=>4},
+      {"count"=>15, "min"=>11, "max"=>30, "mean"=>70, "sum"=>1050, "median"=>8, "stddev"=>5},
+      {"count"=>18, "min"=>10, "max"=>30, "mean"=>65, "sum"=>1170, "median"=>9, "stddev"=>6},
+      {"count"=>21, "min"=>9, "max"=>30, "mean"=>60, "sum"=>1260, "median"=>10, "stddev"=>7},
+      {"count"=>24, "min"=>8, "max"=>30, "mean"=>55, "sum"=>1320, "median"=>11, "stddev"=>8},
+      {"count"=>27, "min"=>7, "max"=>30, "mean"=>50, "sum"=>1350, "median"=>12, "stddev"=>9},
+      {"count"=>30, "min"=>6, "max"=>30, "mean"=>40, "sum"=>1200, "median"=>13, "stddev"=>10},
+      {"count"=>33, "min"=>16, "max"=>30, "mean"=>30, "sum"=>990, "median"=>14, "stddev"=>11},
+      {"count"=>36, "min"=>17, "max"=>30, "mean"=>20, "sum"=>720, "median"=>15, "stddev"=>12},
+      {"count"=>39, "min"=>18, "max"=>30, "mean"=>10, "sum"=>390, "median"=>16, "stddev"=>13},
+      {"count"=>42, "min"=>19, "max"=>30, "mean"=>5, "sum"=>210, "median"=>17, "stddev"=>14},
+      {"count"=>45, "min"=>20, "max"=>30, "mean"=>1, "sum"=>45, "median"=>18, "stddev"=>15}
+    ]
+    i = 0
+    (@start_timestamp..@end_timestamp).each do |t|
+      col.update({"t" => t}, {"$set" => {"m.test" => timers[i]}}, :upsert => true)
+      i += 1
+    end
+    
+    seq = Mouth::Sequence.new("test.test", :kind => :timer, :granularity_in_minutes => 15, :start_time => @start_time, :end_time => @end_time)
+    values = seq.sequence
+    
+    assert_equal 1, values.length
+    
+    value = values.first
+    
+    expected_count = timers.collect {|t| t["count"] }.inject(0) {|s,c| s + c }
+    expected_sum = timers.collect {|t| t["sum"] }.inject(0) {|s,c| s + c }
+    
+    assert_equal expected_count, value["count"]
+    assert_equal expected_sum, value["sum"]
+    assert_equal timers.collect {|t| t["min"] }.min, value["min"]
+    assert_equal timers.collect {|t| t["max"] }.max, value["max"]
+    assert_equal expected_sum / expected_count, value["mean"].to_i
+    assert_equal 11, value["median"]
+    assert_equal 29, value["stddev"].to_i # NOTE: i don't actually know if this is correct
+    
+  end
+  
+  def test_empty_timer
+    @start_time = Time.new(2012, 4, 1, 9, 30, 0, "-07:00")
+    @end_time = Time.new(2012, 4, 1, 9, 44, 0, "-07:00")
+    
+    @start_timestamp = @start_time.to_i / 60
+    @end_timestamp = @end_time.to_i / 60
+    
+    col = Mouth.collection(Mouth.mongo_collection_name("test"))
+    col.remove
+    seq = Mouth::Sequence.new("test.test", :kind => :timer, :granularity_in_minutes => 15, :start_time => @start_time, :end_time => @end_time)
+    values = seq.sequence
+    
+    assert_equal [{"count"=>0,
+                   "min"=>nil,
+                   "max"=>nil,
+                   "mean"=>nil,
+                   "sum"=>0,
+                   "median"=>nil,
+                   "stddev"=>nil}], values
+  end
+  
+  def test_empty_sequence
+    @start_time = Time.new(2012, 4, 1, 9, 30, 0, "-07:00")
+    @end_time = Time.new(2012, 4, 1, 9, 44, 0, "-07:00")
+    
+    @start_timestamp = @start_time.to_i / 60
+    @end_timestamp = @end_time.to_i / 60
+    
+    col = Mouth.collection(Mouth.mongo_collection_name("test"))
+    col.remove
+    seq = Mouth::Sequence.new("test.test", :kind => :counter, :granularity_in_minutes => 15, :start_time => @start_time, :end_time => @end_time)
+    values = seq.sequence
+    
+    assert_equal [0], values
+  end
+  
+end