mouth 0.8.1 → 0.8.2

data/lib/mouth/runner.rb

@@ -12,8 +12,6 @@ module Mouth
     attr_accessor :options
     
     def initialize(opts={})
-      puts "Starting Mouth..."
-      
       self.log_file = opts[:log_file]
       self.pid_file = opts[:pid_file]
       self.verbosity = opts[:verbosity]
@@ -23,6 +21,8 @@ module Mouth
     def run!
       kill! if self.options[:kill]
       
+      puts "Starting Mouth..."
+      
       daemonize!
       save_pid!
       setup_logging!

data/lib/mouth/{sequence.rb → sequence_query.rb}

@@ -1,16 +1,16 @@
 module Mouth
   
   # Usage: 
-  # Sequence.new(["namespace.foobar_occurances"]).sequences
+  # SequenceQuery.new(["namespace.foobar_occurances"]).sequences
   # # => {"foobar_occurances" => [4, 9, 0, ...]}
   #
-  # Sequence.new(["namespace.foobar_occurances", "namespace.baz"], :kind => :timer).sequences
+  # SequenceQuery.new(["namespace.foobar_occurances", "namespace.baz"], :kind => :timer).sequences
   # # => {"foobar_occurances" => [{:count => 3, :min => 1, ...}, ...], "baz" => [...]}
   #
-  # s = Sequence.new(...)
+  # s = SequenceQuery.new(...)
   # s.time_sequence
   # # => [Time.new(first datapoint), Time.new(second datapoint), ..., Time.new(last datapoint)]
-  class Sequence
+  class SequenceQuery
     
     attr_accessor :keys
     attr_accessor :kind
@@ -63,10 +63,24 @@ module Mouth
     end
     
     def time_sequence
+      if self.granularity_in_minutes == 1
+        start_timestamp = self.start_time.to_i / 60
+        end_timestamp = self.end_time.to_i / 60
+      else
+        start_timestamp = timestamp_to_nearest(self.start_time, self.granularity_in_minutes, :down)
+        end_timestamp = timestamp_to_nearest(self.end_time, self.granularity_in_minutes, :up) - 1
+      end
+      
+      seq = []
+      (start_timestamp..end_timestamp).step(self.granularity_in_minutes) do |ts|
+        seq << Time.at(ts * 60)
+      end
+      seq
     end
     
     # Epoch in seconds
     def epoch_sequence
+      time_sequence.collect(&:to_i)
     end
     
     protected
@@ -79,8 +93,8 @@ module Mouth
       ###
       map_function = <<-JS
         function() {
-          var doc = {t: this.t, m: {}, c: {}}
-          ,   thisMetrics = this.#{self.kind_letter.to_s}
+          var doc = {t: this.t, m: {}, c: {}, g: {}}
+          ,   thisMetrics = this.#{self.kind_letter.to_s} || {}
           ,   docMetrics = doc.#{self.kind_letter.to_s}
           ,   fields = #{self.metrics.to_s}
           ,   i, k, val
@@ -99,7 +113,7 @@ module Mouth
   
       reduce_function = <<-JS
         function(key, values) {
-          var result = {c: {}, m: {}}
+          var result = {c: {}, m: {}, g: {}}
           ,   k
           ,   existing
           ,   current
@@ -109,12 +123,20 @@ module Mouth
             if (value.c) {
               for (k in value.c) {
                 existing = result.c[k] || 0;
-                result.c[k] = existing + value.c[k]
+                result.c[k] = existing + value.c[k];
+              }
+            }
+            if (value.g) {
+              for (k in value.g) {
+                existing = result.g[k] || [0, -1];
+                if (value.t > existing[1]) {
+                  result.g[k] = [value.g[k], value.t];
+                }
               }
             }
             if (value.m) {
               for (k in value.m) {
-                current = value.m[k]
+                current = value.m[k];
                 existing = result.m[k];
                 if (!existing) {
                   current.median = [current.median];
@@ -140,6 +162,10 @@ module Mouth
             }
           });
           
+          for (k in result.g) {
+            result.g[k] = result.g[k][0];
+          }
+          
           for (k in result.m) {
             existing = result.m[k];
             
@@ -216,7 +242,14 @@ module Mouth
         h
       end
       
-      default = self.kind == :counter ? 0 : {"count" => 0, "min" => nil, "max" => nil, "mean" => nil, "sum" => 0, "median" => nil, "stddev" => nil}
+      default = case self.kind
+      when :counter
+        0
+      when :timer
+        {"count" => 0, "min" => nil, "max" => nil, "mean" => nil, "sum" => 0, "median" => nil, "stddev" => nil}
+      when :gauge
+        nil
+      end
       
       seqs = {}
       self.metrics.each do |m|
@@ -228,15 +261,29 @@ module Mouth
         seqs[m] = seq
       end
       
+      if self.kind == :gauge
+        seqs.each_pair do |met, seq|
+          cur_val = nil
+          seq.each_with_index do |v, i|
+            seq[i] = cur_val || 0 unless v  # Note: In the future, we can remove || 0 and have leading nils.  Then, fill those in with values from a previous time period
+            cur_val = v || cur_val
+          end
+        end
+      end
+      
       seqs
     end
     
     def collection
-      @collection ||= Mouth.collection(Mouth.mongo_collection_name(self.namespace))
+      @collection ||= Mouth.collection_for(self.namespace)
     end
     
     def kind_letter
-      @kind_letter ||= self.kind == :counter ? "c" : "m"
+      @kind_letter ||= case self.kind
+      when :counter then "c"
+      when :timer then "m"
+      when :gauge then "g"
+      end
     end
     
     def fields
@@ -261,13 +308,14 @@ module Mouth
       opts = {
         :namespace => "sample",
         :metric => "sample",
-        :start_time => (Time.now.to_i / 60 - 300),
-        :end_time => (Time.now.to_i / 60),
+        :start_time => (Mouth.current_timestamp - 300),
+        :end_time => (Mouth.current_timestamp / 60),
       }.merge(opts)
       
-      collection_name = Mouth.mongo_collection_name(opts[:namespace])
+      collection = Mouth.collection_for(opts[:namespace])
       
       counter = 99
+      gauge = 50
       (opts[:start_time]..opts[:end_time]).each do |t|
         
         # Generate garbage data for the sample
@@ -284,12 +332,19 @@ module Mouth
           "stddev" => rand(10)
         }
         
+        set = {"c.#{opts[:metric]}" => counter, "m.#{opts[:metric]}" => m_doc}
+        if rand(4) == 0
+          set["g.#{opts[:metric]}"] = gauge
+        end
+        
         # Insert the document into mongo
-        Mouth.collection(collection_name).update({"t" => t}, {"$set" => {"c.#{opts[:metric]}" => counter, "m.#{opts[:metric]}" => m_doc}}, :upsert => true)
+        collection.update({"t" => t}, {"$set" => set}, :upsert => true)
         
         # Update counter randomly
         counter += rand(10) - 5
+        gauge += rand(10) - 5
         counter = 0 if counter < 0
+        gauge = 0 if gauge < 0
       end
       
       true

data/lib/mouth/source.rb

@@ -16,15 +16,16 @@ module Mouth
       tuples = []
       col_names.each do |col_name|
         namespace = col_name.match(/mouth_(.+)/)[1]
-        counters, timers = all_for_collection(Mouth.collection(col_name))
+        counters, timers, gauges = all_for_collection(Mouth.collection(col_name))
         counters.each {|s| tuples << {:source => "#{namespace}.#{s}", :kind => "counter"} }
         timers.each {|s| tuples << {:source => "#{namespace}.#{s}", :kind => "timer"} }
+        gauges.each {|s| tuples << {:source => "#{namespace}.#{s}", :kind => "gauge"} }
       end
       
       tuples.sort_by {|t| "#{t[:kind]}#{t[:source]}" }.collect {|t| new(t) }
     end
     
-    def self.all_for_collection(col, window = Time.now.to_i / 60 - 120)
+    def self.all_for_collection(col, window = Mouth.current_timestamp - 120)
       map_function = <<-JS
         function() {
           var k, vh;
@@ -38,6 +39,11 @@ module Mouth
             vh[k] = true;
             emit("timers", {ks: vh});
           }
+          for (k in this.g) {
+            vh = {};
+            vh[k] = true;
+            emit("gauges", {ks: vh});
+          }
         }
       JS
       
@@ -58,6 +64,7 @@ module Mouth
       
       counters = []
       timers = []
+      gauges = []
       if result && result["results"].is_a?(Array)
         result["results"].each do |r|
           k, v = r["_id"], r["value"]
@@ -65,11 +72,13 @@ module Mouth
             timers << v["ks"].keys
           elsif k == "counters"
             counters << v["ks"].keys
+          elsif k == "gauges"
+            gauges << v["ks"].keys
           end
         end
       end
       
-      [counters.flatten.compact.uniq, timers.flatten.compact.uniq]
+      [counters.flatten.compact.uniq, timers.flatten.compact.uniq, gauges.flatten.compact.uniq]
     end
     
   end

data/lib/mouth/sucker.rb

@@ -20,15 +20,16 @@ module Mouth
     attr_accessor :port
 
     # Actual EM::Mongo connection
-    attr_accessor :mongo_db
+    attr_accessor :mongo
     
     # Info to connect to mongo
     attr_accessor :mongo_db_name
-    attr_accessor :mongo_hosts
+    attr_accessor :mongo_hostports
     
     # Accumulators of our data
     attr_accessor :counters
     attr_accessor :timers
+    attr_accessor :gauges
     
     # Stats
     attr_accessor :udp_packets_received
@@ -38,19 +39,28 @@ module Mouth
       self.host = options[:host] || "localhost"
       self.port = options[:port] || 8889
       self.mongo_db_name = options[:mongo_db_name] || "mouth"
-      self.mongo_hosts = options[:mongo_hosts] || ["localhost"]
+      hostports = options[:mongo_hostports] || [["localhost", EM::Mongo::DEFAULT_PORT]]
+      self.mongo_hostports = hostports.collect do |hp|
+        if hp.is_a?(String)
+          host, port = hp.split(":")
+          [host, port || EM::Mongo::DEFAULT_PORT]
+        else
+          hp
+        end
+      end
       
       self.udp_packets_received = 0
       self.mongo_flushes = 0
       
       self.counters = {}
       self.timers = {}
+      self.gauges = {}
     end
     
     def suck!
       EM.run do
         # Connect to mongo now
-        self.mongo_db
+        self.mongo
 
         EM.open_datagram_socket host, port, SuckerConnection do |conn|
           conn.sucker = self
@@ -59,6 +69,7 @@ module Mouth
         EM.add_periodic_timer(10) do
           Mouth.logger.info "Counters: #{self.counters.inspect}"
           Mouth.logger.info "Timers: #{self.timers.inspect}"
+          Mouth.logger.info "Gauges: #{self.gauges.inspect}"
           self.flush!
           self.set_procline!
         end
@@ -73,7 +84,7 @@ module Mouth
     # counter: gorets:1|c
     # counter w/ sampling: gorets:1|c|@0.1
     # timer: glork:320|ms
-    # (future) gauge: gaugor:333|g
+    # gauge: gaugor:333|g
     def store!(data)
       key_value, command_sampling = data.to_s.split("|", 2)
       key, value = key_value.to_s.split(":")
@@ -84,7 +95,7 @@ module Mouth
       key = Mouth.parse_key(key).join(".")
       value = value.to_f
       
-      ts = minute_timestamps
+      ts = Mouth.current_timestamp
       
       if command == "ms"
         self.timers[ts] ||= {}
@@ -99,13 +110,16 @@ module Mouth
         self.counters[ts] ||= {}
         self.counters[ts][key] ||= 0.0
         self.counters[ts][key] += value * factor
+      elsif command == "g"
+        self.gauges[ts] ||= {}
+        self.gauges[ts][key] = value
       end
       
       self.udp_packets_received += 1
     end
     
     def flush!
-      ts = minute_timestamps
+      ts = Mouth.current_timestamp
       limit_ts = ts - 1
       mongo_docs = {}
       
@@ -117,7 +131,8 @@ module Mouth
       #   },
       #   m: {
       #     occasions: {...}
-      #   }
+      #   },
+      #   g: {things: 3}
       # }
       
       self.counters.each do |cur_ts, counters_to_save|
@@ -136,6 +151,22 @@ module Mouth
         end
       end
       
+      self.gauges.each do |cur_ts, gauges_to_save|
+        if cur_ts <= limit_ts
+          gauges_to_save.each do |gauge_key, value|
+            ns, sub_key = Mouth.parse_key(gauge_key)
+            mongo_key = "#{ns}:#{ts}"
+            mongo_docs[mongo_key] ||= {}
+            
+            cur_mongo_doc = mongo_docs[mongo_key]
+            cur_mongo_doc["g"] ||= {}
+            cur_mongo_doc["g"][sub_key] = value
+          end
+          
+          self.gauges.delete(cur_ts)
+        end
+      end
+      
       self.timers.each do |cur_ts, timers_to_save|
         if cur_ts <= limit_ts
           timers_to_save.each do |timer_key, values|
@@ -160,21 +191,21 @@ module Mouth
       
       mongo_docs.each do |key, doc|
         ns, ts = key.split(":")
-        collection_name = "mouth_#{ns}"
+        collection_name = Mouth.mongo_collection_name(ns)
         doc["t"] = ts.to_i
         
-        self.mongo_db.collection(collection_name).insert(doc)
+        self.mongo.collection(collection_name).insert(doc)
       end
       
       self.mongo_flushes += 1 if mongo_docs.any?
     end
     
-    def mongo_db
-      @mongo_db ||= begin
-        if self.mongo_hosts.length == 1
-          EM::Mongo::Connection.new(self.mongo_hosts.first).db(self.mongo_db_name)
+    def mongo
+      @mongo ||= begin
+        if self.mongo_hostports.length == 1
+          EM::Mongo::Connection.new(*self.mongo_hostports.first).db(self.mongo_db_name)
         else
-          raise "TODO: ability to connect to a replica set."
+          raise "Ability to connect to a replica set not implemented."
         end
       end
     end
@@ -185,10 +216,6 @@ module Mouth
     
     private
     
-    def minute_timestamps
-      Time.now.to_i / 60
-    end
-    
     def analyze_timer(values)
       values.sort!
       

data/lib/mouth/version.rb

@@ -1,3 +1,3 @@
 module Mouth
-  VERSION = "0.8.1"
+  VERSION = "0.8.2"
 end

data/test/instrument_test.rb

@@ -0,0 +1,79 @@
+$LOAD_PATH.unshift 'test'
+require 'test_helper'
+
+require 'mouth/instrument'
+
+class MouthInstrumentTest < Test::Unit::TestCase
+  def setup
+    @@udp ||= UDPSocket.new.tap do |u|
+      u.bind("127.0.0.1", 32123)
+    end
+    @udp = @@udp
+    100.times { @udp.recvfrom_nonblock(1024) } rescue nil
+    Mouth.daemon_hostport = "127.0.0.1:32123"
+  end
+  
+  def test_increment
+    Mouth.increment("bob")
+    
+    res =  @udp.recvfrom(1024)
+    assert_equal "bob:1|c", res[0]
+  end
+  
+  def test_increment_delta
+    Mouth.increment("bob.rob", 123)
+    
+    res =  @udp.recvfrom(1024)
+    assert_equal "bob.rob:123|c", res[0]
+  end
+  
+  def test_increment_sample
+    pack = nil
+    nothing = false
+     
+    100.times do
+      Mouth.increment("bob.rob/blah-haha", 123, 0.1)
+      
+      begin
+        pack ||= @udp.recvfrom_nonblock(1024)
+        break if nothing
+      rescue IO::WaitReadable
+        nothing = true
+      end
+    end
+    
+    assert nothing
+    assert_equal "bob.rob/blah-haha:123|c|@0.1", pack[0]
+  end
+  
+  def test_gauge
+    Mouth.gauge("bob", 77)
+    
+    res =  @udp.recvfrom(1024)
+    assert_equal "bob:77|g", res[0]
+  end
+  
+  def test_measure
+    Mouth.measure("bob", 1.2)
+    
+    res =  @udp.recvfrom(1024)
+    assert_equal "bob:1.2|ms", res[0]
+  end
+  
+  def test_measure_block
+    result = Mouth.measure("bob") do
+      sleep 0.2
+      9876
+    end
+    
+    assert_equal 9876, result
+    
+    res =  @udp.recvfrom(1024)[0]
+    m = res.match(/bob:(\d+)\|ms/)
+    assert m
+    
+    # Make sure the number is within a resonable range
+    assert m[1].to_i > 180
+    assert m[1].to_i < 220
+  end
+end