simple_metrics 0.3.6 → 0.4.0

data/lib/simple_metrics/server.rb

@@ -0,0 +1,66 @@
+# encoding: utf-8
+require "eventmachine"
+
+module SimpleMetrics
+
+  module ClientHandler
+
+    @@data_points = []
+
+    class << self
+      def get_and_clear_data_points
+        data_points = @@data_points.dup
+        @@data_points = []
+        data_points
+      end
+    end
+
+    def data_points
+      @@data_points
+    end
+
+    def post_init
+      SimpleMetrics.logger.info "ClientHandler entering post_init"
+    end
+
+    def receive_data(data)
+      SimpleMetrics.logger.debug "received_data: #{data.inspect}"
+
+      @@data_points ||= []
+      @@data_points << DataPoint.parse(data)
+    rescue DataPoint::ParserError => e
+      SimpleMetrics.logger.debug "Invalid Data skipped: #{data}"
+    end
+  end
+
+  class Server
+
+    attr_reader :db, :connection
+
+    def start
+      SimpleMetrics.logger.info "SERVER: starting up on #{SimpleMetrics.config[:host]}:#{SimpleMetrics.config[:port]}..."
+
+      Mongo.ensure_collections_exist
+
+      EM.run do
+        EM.open_datagram_socket(SimpleMetrics.config[:host], SimpleMetrics.config[:port], SimpleMetrics::ClientHandler) do |con|
+          EventMachine::add_periodic_timer(SimpleMetrics.config[:flush_interval]) do
+            SimpleMetrics.logger.debug "SERVER: period timer triggered after #{SimpleMetrics.config[:flush_interval]} seconds"
+
+            EM.defer { Bucket.flush_data_points(ClientHandler.get_and_clear_data_points) } 
+          end
+        end
+      end
+    end
+
+    def stop
+      SimpleMetrics.logger.info "EventMachine stop"
+      EM.stop
+    end
+    
+    def to_s
+      "#{SimpleMetrics.config[:host]}:#{SimpleMetrics.config[:port]}"
+    end
+
+  end
+end

data/lib/simple_metrics/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module SimpleMetrics
-  VERSION = "0.3.6"
+  VERSION = "0.4.0"
 end

data/simple_metrics.gemspec

@@ -19,15 +19,9 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rake"
   s.add_development_dependency "rspec"
   s.add_development_dependency "rr"
-  s.add_development_dependency "shotgun"
 
   s.add_dependency "eventmachine"
   s.add_dependency "daemons"
   s.add_dependency "mongo", '~> 1.6'
   s.add_dependency "bson", '~> 1.6'
-  s.add_dependency "bson_ext", '~> 1.6'
-  s.add_dependency "sinatra"
-  s.add_dependency "erubis"
-  s.add_dependency "vegas", '~> 0.1.2'
-  s.add_dependency "json"
 end

data/spec/bucket_spec.rb

@@ -37,17 +37,17 @@ module SimpleMetrics
 
     describe "#save" do
       before do
-        DataPointRepository.truncate_collections
-        DataPointRepository.ensure_collections_exist
+        Mongo.truncate_collections
+        Mongo.ensure_collections_exist
         bucket.save(stats, ts)
       end
 
       let(:stats) do
-        DataPoint::Counter.new(:name => "key1", :value => 5)
+        DataPoint.create_counter(:name => "key1", :value => 5)
       end
 
       it "saves given data in bucket" do
-        results = bucket.find_all_at_ts(ts)
+        results = bucket.find_all_by_name("key1")
         results.should have(1).item
         result = results.first
         result.name.should  == stats.name
@@ -56,7 +56,7 @@ module SimpleMetrics
       end
 
       it "saves data in correct timestamp" do
-        result = bucket.find_all_at_ts(ts).first
+        result = bucket.find_all_by_name("key1").first
         result.ts.should == ts/sec*sec
       end
 
@@ -65,33 +65,68 @@ module SimpleMetrics
     describe "finder methods" do
 
       before do
-        DataPointRepository.truncate_collections
-        DataPointRepository.ensure_collections_exist
+        Mongo.truncate_collections
+        Mongo.ensure_collections_exist
       end
 
-      describe "#find_all_at_ts" do
+      describe "#find_all_by_name" do
+        it "returns all stats for given name" do
+          stats_same1 =  DataPoint.create_counter(:name => "key1", :value => 5)
+          stats_same2 =  DataPoint.create_counter(:name => "key1", :value => 3)
+          stats_different = DataPoint.create_counter(:name => "key2", :value => 3)
+
+          bucket.save(stats_same1, ts)
+          bucket.save(stats_same2, ts)
+          bucket.save(stats_different, ts)
+
+          results = bucket.find_all_by_name("key1")
+          results.should have(2).items
+          results.first.name.should == stats_same1.name
+        end
+      end
+
+      describe "#find_all_in_ts" do
         it "returns all stats in given timestamp" do
-          stats1  =  DataPoint::Counter.new(:name => "key1", :value => 5)
-          stats2  =  DataPoint::Counter.new(:name => "key2", :value => 3)
+          stats1  =  DataPoint.create_counter(:name => "key1", :value => 5)
+          stats2  =  DataPoint.create_counter(:name => "key2", :value => 3)
 
           bucket.save(stats1, ts)
           bucket.save(stats2, bucket.next_ts_bucket(ts))
 
-          result1 = bucket.find_all_at_ts(ts).first
+          result1 = bucket.find_all_in_ts(ts).first
           result1.name.should == stats1.name
           result1.value.should == stats1.value
 
-          result2 = bucket.find_all_at_ts(bucket.next_ts_bucket(ts)).first
+          result2 = bucket.find_all_in_ts(bucket.next_ts_bucket(ts)).first
           result2.name.should == stats2.name
           result2.value.should == stats2.value
         end
       end
 
-      describe "#find_all_in_ts_range_by_wildcard" do
+      describe "#find_all_in_ts_by_name" do
+        it "returns all stats for given name and timestamp" do
+          stats1a  =  DataPoint.create_counter(:name => "key1", :value => 5)
+          stats1b  =  DataPoint.create_counter(:name => "key1", :value => 7)
+          stats2   =  DataPoint.create_counter(:name => "key2", :value => 7)
+          stats1_different_ts   =  DataPoint.create_counter(:name => "key1", :value => 3)
+
+          bucket.save(stats1a, ts)
+          bucket.save(stats1b, ts)
+          bucket.save(stats2, ts)
+          bucket.save(stats1_different_ts, bucket.next_ts_bucket(ts))
+
+          results = bucket.find_all_in_ts_by_name(ts, "key1")
+          results.should have(2).items
+          results.first.name.should == "key1"
+          results.last.name.should == "key1"
+        end
+      end
+
+      describe "#find_all_in_ts_by_wildcard" do
         it "returns all stats for given name and timestamp" do
-          stats1  =  DataPoint::Counter.new(:name => "com.test.key1", :value => 5)
-          stats2  =  DataPoint::Counter.new(:name => "com.test.key2", :value => 7)
-          stats_different =  DataPoint::Counter.new(:name => "com.test2.key1", :value => 3)
+          stats1  =  DataPoint.create_counter(:name => "com.test.key1", :value => 5)
+          stats2  =  DataPoint.create_counter(:name => "com.test.key2", :value => 7)
+          stats_different =  DataPoint.create_counter(:name => "com.test2.key1", :value => 3)
 
           from = bucket.ts_bucket(ts)
           to   = from
@@ -109,7 +144,7 @@ module SimpleMetrics
 
       describe "#fill_gaps" do
         it "returns stats and fills missing gaps with null entries" do
-          stats  =  DataPoint::Counter.new(:name => "com.test.key1", :value => 5)
+          stats  =  DataPoint.create_counter(:name => "com.test.key1", :value => 5)
 
           from = ts - 10
           to   = ts + 10
@@ -134,5 +169,105 @@ module SimpleMetrics
       end
     end # describe "finder methods"
 
+    describe "#aggregate_all" do
+      before do
+        Mongo.truncate_collections
+        Mongo.ensure_collections_exist
+      end
+
+      it "aggregates all counter data points" do
+        stats1a  =  DataPoint.create_counter(:name => "key1", :value => 5)
+        stats1b  =  DataPoint.create_counter(:name => "key1", :value => 7)
+        stats2   =  DataPoint.create_counter(:name => "key2", :value => 3)
+
+        bucket2 = Bucket[1]
+        ts_at_insert = bucket2.previous_ts_bucket(ts)
+        bucket.save(stats1a, ts_at_insert)
+        bucket.save(stats1b, ts_at_insert)
+        bucket.save(stats2, ts_at_insert)
+
+        Bucket.aggregate_all(ts)
+
+        results = bucket2.find_all_in_ts(ts_at_insert)
+        results.should have(2).items
+
+        key1_result = results.find {|stat| stat.name == "key1"}
+        key1_result.value.should == 12
+        key1_result.should be_counter
+
+        key2_result = results.find {|stat| stat.name == "key2"}
+        key2_result.value.should == 3
+        key2_result.should be_counter
+      end
+
+      it "aggregates all gauge data points" do
+        stats1a  =  DataPoint.create_gauge(:name => "key1", :value => 5)
+        stats1b  =  DataPoint.create_gauge(:name => "key1", :value => 7)
+        stats2   =  DataPoint.create_gauge(:name => "key2", :value => 3)
+
+        bucket2 = Bucket[1]
+        ts_at_insert = bucket2.previous_ts_bucket(ts)
+        bucket.save(stats1a, ts_at_insert)
+        bucket.save(stats1b, ts_at_insert)
+        bucket.save(stats2, ts_at_insert)
+
+        Bucket.aggregate_all(ts)
+
+        results = bucket2.find_all_in_ts(ts_at_insert)
+        results.should have(2).items
+
+        key1_result = results.find {|stat| stat.name == "key1"}
+        key1_result.value.should == 6
+        key1_result.should be_gauge
+
+        key2_result = results.find {|stat| stat.name == "key2"}
+        key2_result.value.should == 3
+        key2_result.should be_gauge
+      end
+
+    end # describe "#aggregate_all"
+
+    describe "#flush_data_points" do
+      before do
+        Mongo.truncate_collections
+        Mongo.ensure_collections_exist
+        
+        stats1 = DataPoint.create_counter(:name => "key1", :value => 5)
+        stats2 = DataPoint.create_counter(:name => "key1", :value => 7)
+        stats3 = DataPoint.create_counter(:name => "key2", :value => 3)
+        @stats = [stats1, stats2, stats3]
+      end
+
+      it "saves all stats in finest/first bucket" do
+        Bucket.flush_data_points(@stats)
+
+        results = bucket.find_all_in_ts(ts)
+        results.should have(2).items
+      end
+
+      it "calls aggregate_all afterwards" do
+        mock(Bucket).aggregate_all(ts)
+        Bucket.flush_data_points(@stats)
+      end
+
+      it "saves all stats and aggregate if duplicates found" do
+        Bucket.flush_data_points(@stats)
+
+        results = bucket.find_all_in_ts(ts)
+        results.should have(2).items
+        results.first.name.should == "key1"
+        results.last.name.should == "key2"
+        results.first.value == 12
+        results.last.value == 3
+      end
+
+      it "raises error if name matches but type does not" do
+        stats4 = DataPoint.create_gauge(:name => "key1", :value => 3)
+        input = @stats + [stats4]
+        expect { Bucket.flush_data_points(input) }.to raise_error(SimpleMetrics::DataPoint::NonMatchingTypesError)
+      end
+
+    end # describe "#flush_data_points"
+
   end
 end

data/spec/data_point_spec.rb

@@ -54,28 +54,78 @@ module SimpleMetrics
       end
 
       it "parses increment timing with sample rate" do
-        # TODO
       end
-    end # describe
+    end
 
-    describe "#aggregate" do
 
-      it "aggregate counters" do
-        key1 = DataPoint::Counter.new(:name => "key1", :value => 1)
-        key2 = DataPoint::Counter.new(:name => "key1", :value => 3)
-        result = DataPoint.aggregate_values([key1, key2])
+    describe "#aggregate" do
+      it "aggregates counter data points" do
+        stats1  =  DataPoint.create_counter(:name => "key1", :value => 5)
+        stats2  =  DataPoint.create_counter(:name => "key1", :value => 7)
+        result = DataPoint.aggregate([stats1, stats2])
+        result.value.should == 12
         result.name.should == "key1"
-        result.value.should == 4
+        result.should be_counter
+      end
+
+      it "aggregates counter data points with custom name" do
+        stats1  =  DataPoint.create_counter(:name => "key1", :value => 5)
+        stats2  =  DataPoint.create_counter(:name => "key1", :value => 7)
+        result = DataPoint.aggregate([stats1, stats2], "new_name")
+        result.value.should == 12
+        result.name.should == "new_name"
+        result.should be_counter
       end
 
-      it "aggregate gauges" do
-        key1 = DataPoint::Gauge.new(:name => "key1", :value => 1)
-        key2 = DataPoint::Gauge.new(:name => "key1", :value => 3)
-        result = DataPoint.aggregate_values([key1, key2])
+      it "aggregates gauge data points" do
+        stats1  =  DataPoint.create_gauge(:name => "key1", :value => 5)
+        stats2  =  DataPoint.create_gauge(:name => "key1", :value => 7)
+        result = DataPoint.aggregate([stats1, stats2])
+        result.value.should == 6
         result.name.should == "key1"
-        result.value.should == 2
+        result.should be_gauge
       end
 
-    end # describe
+      it "aggregates timing data points" do
+      end
+      
+      it "aggregates event data points" do
+      end
+    end
+
+     describe "#aggregate_array" do
+      it "aggregates counter data points" do
+        stats1  =  DataPoint.create_counter(:name => "com.test.key1", :value => 5, :ts => ts)
+        stats2  =  DataPoint.create_counter(:name => "com.test.key1", :value => 7, :ts => ts)
+        stats3  =  DataPoint.create_counter(:name => "com.test.key1", :value => 9, :ts => (ts + 60) )
+
+        results = DataPoint.aggregate_array([stats1, stats2, stats3], "com.test.*")
+        results.should have(2).data_points
+        results.first.name.should == "com.test.*"
+        results.last.name.should  == "com.test.*"
+        results.first.value.should == 12
+        results.last.value.should == 9
+      end
+
+      it "aggregates gauge data points" do
+        stats1  =  DataPoint.create_gauge(:name => "com.test.key1", :value => 5, :ts => ts)
+        stats2  =  DataPoint.create_gauge(:name => "com.test.key1", :value => 7, :ts => ts)
+        stats3  =  DataPoint.create_gauge(:name => "com.test.key1", :value => 9, :ts => (ts + 60) )
+
+        results = DataPoint.aggregate_array([stats1, stats2, stats3], "com.test.*")
+        results.should have(2).data_points
+        results.first.name.should == "com.test.*"
+        results.last.name.should  == "com.test.*"
+        results.first.value.should == 6
+        results.last.value.should == 9
+      end
+     
+      it "raises NonMatchingTypesError if types are different" do
+        stats1  =  DataPoint.create_counter(:name => "com.test.key1", :value => 5, :ts => ts)
+        stats2  =  DataPoint.create_gauge(:name => "com.test.key1", :value => 5, :ts => ts)
+        expect { DataPoint.aggregate_array([stats1, stats2], "com.test.*") }.to raise_error(SimpleMetrics::DataPoint::NonMatchingTypesError)
+      end
+    end
   end
+
 end

data/spec/graph_spec.rb

@@ -14,14 +14,14 @@ module SimpleMetrics
     end
 
   	before do
-  	  DataPointRepository.truncate_collections
-  	  DataPointRepository.ensure_collections_exist
+  	  Mongo.truncate_collections
+  	  Mongo.ensure_collections_exist
   	end
 
     describe "#query" do
 
       it "returns string request data points as is" do
-      	dp1 = DataPoint::Counter.new(:name => "key1", :value => 5)
+      	dp1 = DataPoint.create_counter(:name => "key1", :value => 5)
 
       	bucket.save(dp1, ts)
 
@@ -34,7 +34,7 @@ module SimpleMetrics
       end
 
       it "returns string request data points and fill graps" do
-        dp1 = DataPoint::Counter.new(:name => "key1", :value => 5)
+        dp1 = DataPoint.create_counter(:name => "key1", :value => 5)
 
         bucket.save(dp1, ts)
 
@@ -49,8 +49,8 @@ module SimpleMetrics
       end
 
       it "returns wildcard request data points with aggregate counter" do
-      	dp1 = DataPoint::Counter.new(:name => "com.test.key1", :value => 5)
-      	dp2 = DataPoint::Counter.new(:name => "com.test.key2", :value => 7)
+      	dp1 = DataPoint.create_counter(:name => "com.test.key1", :value => 5)
+      	dp2 = DataPoint.create_counter(:name => "com.test.key2", :value => 7)
 
       	bucket.save(dp1, ts)
       	bucket.save(dp2, ts)
@@ -66,35 +66,24 @@ module SimpleMetrics
       	result.should be_counter
       end
 
-    end # descibe
+      it "returns regexp request data points with aggregate gauge" do
+      	dp1 = DataPoint.create_gauge(:name => "com.test.key1", :value => 5)
+      	dp2 = DataPoint.create_gauge(:name => "com.test.key2", :value => 7)
 
-    describe "#query_all" do
-
-      it "returns request data points as is" do
-        dp1 = DataPoint::Counter.new(:name => "com.test.key1", :value => 5)
-        dp2 = DataPoint::Counter.new(:name => "com.test.key2", :value => 7)
-        dp3 = DataPoint::Counter.new(:name => "com.test.key2", :value => 3)
-        bucket.save(dp1, ts)
-        bucket.save(dp2, ts - 10)
-        bucket.save(dp3, ts)
+      	bucket.save(dp1, ts)
+      	bucket.save(dp2, ts)
 
         current_ts = bucket.ts_bucket(ts)
-        from = current_ts - 10
+        from = current_ts
         to   = current_ts
-        results = Graph.query_all(bucket, from, to, "com.test.key1", "com.test.key2")
-        key1 = results.first
-        key1[:name].should == "com.test.key1"
-        key1[:data].should have(2).entry
-        key1[:data].first[:y].should == 0
-        key1[:data].last[:y].should == 5
-
-        key2 = results.last
-        key2[:name].should == "com.test.key2"
-        key2[:data].should have(2).entry
-        key2[:data].first[:y].should == 7
-        key2[:data].last[:y].should == 3
-      end
 
+      	results = Graph.query(bucket, from, to, /com\.test(.*)/)
+      	results.should have(1).data_point
+      	result = results.first
+      	result.name.should == "/com\\.test(.*)/"
+      	result.value.should == 6
+      	result.should be_gauge
+      end
     end
   end
 end