simple_metrics 0.0.1 → 0.2.2

data/bin/populate

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "bundler/setup"
+
+require 'optparse'
+require "simple_metrics"
+
+ts = Time.now.to_i
+
+name = "test.post.clicks2"
+
+SimpleMetrics::Mongo.ensure_collections_exist
+
+SimpleMetrics::Bucket.all.each do |bucket|
+	ts_bucket = bucket.ts_bucket(ts)
+
+	previous_value = rand(300)
+	(1..100).each do |index|
+		ts = ts_bucket - (bucket.seconds*index)
+		value = previous_value+rand(20)
+		data_point = SimpleMetrics::DataPoint.create_counter(:name => name, :value => value)
+		bucket.save(data_point, ts)
+	end
+end

data/lib/simple_metrics/bucket.rb

@@ -21,14 +21,17 @@ module SimpleMetrics
         Bucket.all.sort_by! { |r| r.seconds }[1..-1]
       end
 
-      def flush_stats(stats)
-        return if stats.empty?
-        SimpleMetrics.logger.info "#{Time.now} Flushing #{stats.count} counters to MongoDB"
+      def flush_data_points(data_points)
+        return if data_points.empty?
+        SimpleMetrics.logger.info "#{Time.now} Flushing #{data_points.count} counters to MongoDB"
 
         ts = Time.now.utc.to_i
         bucket = Bucket.first
-        stats.each { |data| bucket.save(data, ts) }
-        
+        data_points.group_by { |data| data.name }.each_pair do |name,dps|
+          data = DataPoint.aggregate(dps)
+          bucket.save(data, ts)
+        end
+
         self.aggregate_all(ts)
       end
 
@@ -41,10 +44,10 @@ module SimpleMetrics
           SimpleMetrics.logger.debug "Aggregating #{bucket.name} #{previous_ts}....#{current_ts} (#{humanized_timestamp(previous_ts)}..#{humanized_timestamp(current_ts)})"
 
           unless bucket.stats_exist_in_previous_ts?(previous_ts)
-            stats_coll = self.first.find_all_in_ts_range(previous_ts, current_ts)
-            stats_coll.group_by { |stats| stats.name }.each_pair do |name,stats_array|
-              stats = Stats.aggregate(stats_array)
-              bucket.save(stats, previous_ts)
+            data_points = self.first.find_all_in_ts_range(previous_ts, current_ts)
+            data_points.group_by { |data| data.name }.each_pair do |name,dps|
+              data = DataPoint.aggregate(dps)
+              bucket.save(data, previous_ts)
             end
           end
         end
@@ -88,33 +91,54 @@ module SimpleMetrics
 
     def find(id)
       mongo_result = mongo_coll.find_one({ :_id => id })
-      Stats.create_from_db(mongo_result)
+      DataPoint.create_from_db(mongo_result)
     end
 
     def find_all_by_name(name)
-      mongo_result = mongo_coll.find({ :name => name })
-      mongo_result.inject([]) { |result, a| result << Stats.create_from_db(a) }
+      mongo_result = mongo_coll.find({ :name => name }).to_a
+      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
     end
 
     def find_all_in_ts(ts)
-      mongo_result = mongo_coll.find({ :ts => ts_bucket(ts) })
-      mongo_result.inject([]) { |result, a| result << Stats.create_from_db(a) }
+      mongo_result = mongo_coll.find({ :ts => ts_bucket(ts) }).to_a
+      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
     end
 
     def find_all_in_ts_by_name(ts, name)
-      mongo_result = mongo_coll.find({ :ts => ts_bucket(ts), :name => name })
-      mongo_result.inject([]) { |result, a| result << Stats.create_from_db(a) }
+      mongo_result = mongo_coll.find({ :ts => ts_bucket(ts), :name => name }).to_a
+      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
     end
 
-    def find_all_in_ts_range(previous_ts, current_ts)
-      mongo_result = mongo_coll.find({ :ts => { "$gte" => previous_ts, "$lt" => current_ts }}).to_a
-      mongo_result.inject([]) { |result, a| result << Stats.create_from_db(a) }
+    def find_all_in_ts_range(from, to)
+      mongo_result = mongo_coll.find({ :ts => { "$gte" => from, "$lte" => to }}).to_a
+      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
+    end
+
+    def find_all_in_ts_range_by_name(from, to, name)
+      mongo_result = mongo_coll.find({ :name => name, :ts => { "$gte" => from, "$lte" => to }}).to_a
+      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
+    end
+
+    def find_all_in_ts_range_by_wildcard(from, to, target)
+      target = target.gsub('.', '\.')
+      target = target.gsub('*', '.*')
+      mongo_result = mongo_coll.find({ :name => /#{target}/, :ts => { "$gte" => from, "$lte" => to } }).to_a
+      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
+    end
+
+    def find_all_in_ts_range_by_regexp(from, to, target)
+      mongo_result = mongo_coll.find({ :name => /#{target}/, :ts => { "$gte" => from, "$lte" => to } }).to_a
+      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
     end
 
     def stats_exist_in_previous_ts?(ts)
       mongo_coll.find({ :ts => ts }).count > 0
     end
 
+    def find_all_distinct_names
+      mongo_coll.distinct(:name).to_a
+    end
+
     def save(stats, ts)
       stats.ts = ts_bucket(ts)
       result = mongo_coll.insert(stats.attributes)
@@ -129,5 +153,36 @@ module SimpleMetrics
       @capped == true
     end
 
+    def fill_gaps(from, to, query_result)
+      return query_result if query_result.nil? || query_result.size == 0
+      
+      tmp_hash = DataPoint.ts_hash(query_result)
+      dp_template = query_result.first
+
+      result = []
+      each_ts(from, to) do |current_bucket_ts|
+        result <<  
+          if tmp_hash.key?(current_bucket_ts)
+            tmp_hash[current_bucket_ts]
+          else
+            dp       = dp_template.dup
+            dp.value = nil
+            dp.ts    = current_bucket_ts
+            dp
+          end
+      end
+      result
+    end
+
+    private
+
+    def each_ts(from, to)
+      current_bucket_ts = ts_bucket(from)
+      while (current_bucket_ts <= ts_bucket(to))
+        yield(current_bucket_ts)
+        current_bucket_ts = current_bucket_ts + seconds
+      end
+    end
+
   end
 end

data/lib/simple_metrics/data_point.rb

@@ -0,0 +1,162 @@
+# encoding: utf-8
+module SimpleMetrics
+
+  class DataPoint
+
+    class NonMatchingTypesError < Exception; end
+    class ParserError           < Exception; end
+
+    # examples:
+    # com.example.test1:1|c
+    # com.example.test2:-1|c
+    # com.example.test2:50|g
+    # com.example.test3:5|c|@0.1
+    # com.example.test4:44|ms
+    REGEXP = /^([\d\w_.]*):(-?[\d]*)\|(c|g|ms){1}(\|@([.\d]+))?$/i
+
+    class << self
+
+      def parse(str)
+        if str =~ REGEXP
+          name, value, type, sample_rate = $1, $2, $3, $5
+          if type == "ms"
+            # TODO: implement sample_rate handling
+            create_timing(:name => name, :value => value)
+          elsif type == "g"
+            create_gauge(:name => name, :value => (value.to_i || 1) * (1.0 / (sample_rate || 1).to_f) )
+          elsif type == "c"
+            create_counter(:name => name, :value => (value.to_i || 1) * (1.0 / (sample_rate || 1).to_f) )
+          end
+        else
+          raise ParserError, "Parser Error - Invalid Stat: #{str}"
+        end
+      end
+
+      def create_counter(attributes)
+        self.new(attributes.merge(:type => 'c'))
+      end
+
+      def create_gauge(attributes)
+        self.new(attributes.merge(:type => 'g'))
+      end
+
+      def create_timing(attributes)
+        self.new(attributes.merge(:type => 'ms'))
+      end
+
+      def aggregate(stats_array, name = nil)
+        raise NonMatchingTypesError unless stats_array.group_by { |stats| stats.type }.size == 1
+
+        result_stat      = stats_array.first.dup
+        result_stat.name = name if name
+        if stats_array.first.counter?
+          result_stat.value = stats_array.map { |stats| stats.value }.inject(0) { |result, value| result += value }
+          result_stat
+        elsif stats_array.first.gauge?
+          total_value = stats_array.map { |stats| stats.value }.inject(0) { |result, value| result += value }
+          result_stat.value = total_value / stats_array.size
+          result_stat
+        elsif stats_array.first.timing?
+          # TODO implement timing aggregation
+        elsif stats_array.first.event? 
+          # TODO implement event aggregation
+        else
+          raise ArgumentError, "Unknown data point type"
+        end
+      end
+
+      def aggregate_array(stats_array, name = nil)
+        raise NonMatchingTypesError unless stats_array.group_by { |stats| stats.type }.size == 1
+
+        if stats_array.first.counter?
+          tmp_hash = ts_hash_aggregated(stats_array) do |value1, value2|
+            value1 + value2
+          end
+
+          result = []
+          tmp_hash.each_pair do |key, value|
+            result << self.new(:name => name, :ts => key, :value => value, :type => 'c')
+          end
+          result
+        elsif stats_array.first.gauge?
+          tmp_hash = ts_hash_aggregated(stats_array) do |value1, value2|
+            (value1 + value2)/2
+          end
+
+          result = []
+          tmp_hash.each_pair do |key, value|
+            result << self.new(:name => name, :ts => key, :value => value, :type => 'g')
+          end
+          result
+        elsif stats_array.first.timing?
+          # TODO implement timing aggregation
+        elsif stats_array.first.event? 
+          # TODO implement event aggregation
+        else
+          raise ArgumentError, "Unknown data point type"
+        end
+      end
+
+      def create_from_db(attributes)
+        self.new(:name => attributes["name"], :value => attributes["value"], :ts => attributes["ts"], :type => attributes["type"])
+      end
+
+      def ts_hash(query_result)
+        query_result.inject({}) { |result, dp| result[dp.ts] = dp; result }
+      end
+      
+      private
+
+      def ts_hash_aggregated(data_points, &block)
+        tmp = {}
+        data_points.each do |dp|
+          if tmp.key?(dp.ts)
+            tmp[dp.ts] = block.call(tmp[dp.ts], dp.value)
+          else
+            tmp[dp.ts] = dp.value
+          end
+        end
+        tmp
+      end
+    end
+
+    attr_accessor :name, :ts, :type, :value
+
+    def initialize(attributes)
+      @name  = attributes[:name]
+      @value = attributes[:value]
+      @ts    = attributes[:ts]
+      @type  = attributes[:type]
+    end
+
+    def counter?
+      type == 'c'
+    end
+
+    def gauge?
+      type == 'g'
+    end
+
+    def timing?
+      type == 'ms'
+    end
+
+    def timestamp
+      ts
+    end
+
+    def value
+      @value.to_i if @value
+    end
+
+    def attributes
+      { 
+        :name  => name,
+        :value => value,
+        :ts    => ts,
+        :type  => type
+      }
+    end
+
+  end
+end

data/lib/simple_metrics/functions.rb

@@ -0,0 +1,70 @@
+module SimpleMetrics
+
+	module Functions
+
+		# calculate the maximum value for multiple targets
+		#
+		# params:
+    #  data_points   [1, 3, 5], [2, 1, 6]
+    #
+    # return:
+    #  array         [2, 3, 6]
+		def max(*data_points)
+    end
+
+    # calculate the minimum value for multiple targets
+    def min(*data_points)
+    end
+
+    # add offset to each value
+		def offset(*data_points)
+    end
+
+    # multiple each value 
+    def scale(*data_points)
+    end
+
+    # Return sum of all databounds
+		# 
+		# params:
+    #  data_points   [1, 3, 5], [2, 1, 6]
+    #
+    # return:
+    #  array         [3, 4, 11]
+    def sum(*data_points)
+    end
+
+  	# 
+  	# Other ideas
+  	# 
+    # as_percent:
+    # * calculate percentage of given targets
+    # * sum of all data_points will be used as the total 100% marker
+    # 
+    # average_above (param):
+    # * return data_points with average value above given param
+    #
+    # average_below (param):
+    # * return data_points with average value below given param
+    #
+    # average_series:
+    # * return average value of given multiple targets
+    #
+    # current_above (param):
+    # * return data_points with value above given param
+    #
+    # current_below (param):
+    # * return data_points with value below given param
+    #
+    # derivative:
+    # * take an absolute value based target and show how many requests per min were handled
+    #
+    # integral:
+    # * calculate sum over time for relative values collected per minute
+    #
+    # logarithm:
+    # * calculate the value with log n (base 10 default)
+    # 
+
+	end # module Functions
+end

data/lib/simple_metrics/graph.rb

@@ -0,0 +1,58 @@
+module SimpleMetrics
+
+  # 
+  # url format examples:
+  # * target=com.post.clicks (1 line in graph)
+  # * target=com.post.clicks.text&target=com.post.clicks.logo (2 lines in graph)
+  # * target=com.post.clicks.* (1 aggregated line in graph)
+  #
+  module Graph
+    extend self
+
+    def minutes
+      Bucket[0]
+    end
+
+    def hours
+      Bucket[1]
+    end
+
+    def day
+      Bucket[2]
+    end
+
+    def week
+      Bucket[3]
+    end
+
+    def query_all(bucket, from, to, *targets)
+      result = {}
+      Array(targets).each do |target|
+        result[target.inspect] = values_only(query(bucket, from, to, target))
+      end
+      result
+    end
+
+    def query(bucket, from, to, target)
+      if target.is_a?(Regexp) 
+        result = bucket.find_all_in_ts_range_by_regexp(from, to, target)
+        result = DataPoint.aggregate_array(result, target.inspect)
+        bucket.fill_gaps(from, to, result)
+      elsif target.is_a?(String) && target.include?('*')
+        result = bucket.find_all_in_ts_range_by_wildcard(from, to, target)
+        result = DataPoint.aggregate_array(result, target)
+        bucket.fill_gaps(from, to, result)
+      elsif target.is_a?(String)
+        result = bucket.find_all_in_ts_range_by_name(from, to, target)
+        bucket.fill_gaps(from, to, result) 
+      else
+        raise ArgumentError, "Unknown target: #{target.inspect}"
+      end
+    end
+
+    def values_only(data_point_array)
+      data_point_array.map { |data| { :ts => data.ts, :value => data.value } }
+    end
+
+  end
+end

data/lib/simple_metrics/mongo.rb

@@ -12,6 +12,7 @@ module SimpleMetrics
           db.create_collection(bucket.name, :capped => bucket.capped, :size => bucket.size) 
           SimpleMetrics.logger.debug "SERVER: MongoDB - created collection #{bucket.name}, capped: #{bucket.capped}, size: #{bucket.size}"
         end
+        
         db.collection(bucket.name).ensure_index([['ts', ::Mongo::ASCENDING]])
         SimpleMetrics.logger.debug "SERVER: MongoDB - ensure index on column ts for collection #{bucket.name}"
       end 
@@ -36,7 +37,7 @@ module SimpleMetrics
     end
 
     def connection
-      @@connection ||= ::Mongo::Connection.new(SimpleMetrics.db_config[:host])
+      @@connection ||= ::Mongo::Connection.new(SimpleMetrics.db_config[:host], SimpleMetrics.db_config[:port])
     end
 
     def db

data/lib/simple_metrics/server.rb

@@ -5,18 +5,18 @@ module SimpleMetrics
 
   module ClientHandler
 
-    @@stats = []
+    @@data_points = []
 
     class << self
-      def get_and_clear_stats
-        stats = @@stats.dup
-        @@stats = []
-        stats
+      def get_and_clear_data_points
+        data_points = @@data_points.dup
+        @@data_points = []
+        data_points
       end
     end
 
-    def stats
-      @@stats
+    def data_points
+      @@data_points
     end
 
     def post_init
@@ -26,9 +26,9 @@ module SimpleMetrics
     def receive_data(data)
       SimpleMetrics.logger.debug "received_data: #{data.inspect}"
 
-      @@stats ||= []
-      @@stats << Stats.parse(data)
-    rescue Stats::ParserError => e
+      @@data_points ||= []
+      @@data_points << DataPoint.parse(data)
+    rescue DataPoint::ParserError => e
       SimpleMetrics.logger.debug "Invalid Data skipped: #{data}"
     end
   end
@@ -47,7 +47,7 @@ module SimpleMetrics
           EventMachine::add_periodic_timer(SimpleMetrics.config[:flush_interval]) do
             SimpleMetrics.logger.debug "SERVER: period timer triggered after #{SimpleMetrics.config[:flush_interval]} seconds"
 
-            EM.defer { Bucket.flush_stats(ClientHandler.get_and_clear_stats) } 
+            EM.defer { Bucket.flush_data_points(ClientHandler.get_and_clear_data_points) } 
           end
         end
       end

data/lib/simple_metrics/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module SimpleMetrics
-  VERSION = "0.0.1"
+  VERSION = "0.2.2"
 end

data/lib/simple_metrics.rb

@@ -4,8 +4,10 @@ require "logger"
 require "simple_metrics/version"
 require "simple_metrics/client"
 require "simple_metrics/server"
-require "simple_metrics/stats"
+require "simple_metrics/data_point"
 require "simple_metrics/bucket"
+require "simple_metrics/graph"
+require "simple_metrics/functions"
 require "simple_metrics/mongo"
 
 module SimpleMetrics
@@ -76,12 +78,14 @@ module SimpleMetrics
 
   DB_CONFIG_DEFAULTS = {
     :host      => 'localhost',
+    :port      => 27017,
     :prefix    => 'development'
   }.freeze
 
   def db_config=(options)
     @@db_config = {
-      :host    => options.delete(:host),
+      :host    => options.delete(:host) || 'localhost',
+      :port    => options.delete(:port) || 27017,
       :db_name => "simple_metrics_#{options.delete(:prefix)}",
       :options => MONGODB_DEFAULTS.merge(options)
     }

data/simple_metrics.gemspec

@@ -22,6 +22,6 @@ Gem::Specification.new do |s|
 
   s.add_dependency "eventmachine"
   s.add_dependency "daemons"
-  s.add_dependency "mongo"
-  s.add_dependency "bson_ext"
+  s.add_dependency "mongo", '~> 1.6'
+  s.add_dependency "bson", '~> 1.6'
 end

data/spec/bucket_spec.rb

@@ -43,7 +43,7 @@ module SimpleMetrics
       end
 
       let(:stats) do
-        Stats.create_counter(:name => "key1", :value => 5)
+        DataPoint.create_counter(:name => "key1", :value => 5)
       end
 
       it "saves given data in bucket" do
@@ -71,9 +71,9 @@ module SimpleMetrics
 
       describe "#find_all_by_name" do
         it "returns all stats for given name" do
-          stats_same1 =  Stats.create_counter(:name => "key1", :value => 5)
-          stats_same2 =  Stats.create_counter(:name => "key1", :value => 3)
-          stats_different = Stats.create_counter(:name => "key2", :value => 3)
+          stats_same1 =  DataPoint.create_counter(:name => "key1", :value => 5)
+          stats_same2 =  DataPoint.create_counter(:name => "key1", :value => 3)
+          stats_different = DataPoint.create_counter(:name => "key2", :value => 3)
 
           bucket.save(stats_same1, ts)
           bucket.save(stats_same2, ts)
@@ -87,8 +87,8 @@ module SimpleMetrics
 
       describe "#find_all_in_ts" do
         it "returns all stats in given timestamp" do
-          stats1  =  Stats.create_counter(:name => "key1", :value => 5)
-          stats2  =  Stats.create_counter(:name => "key2", :value => 3)
+          stats1  =  DataPoint.create_counter(:name => "key1", :value => 5)
+          stats2  =  DataPoint.create_counter(:name => "key2", :value => 3)
 
           bucket.save(stats1, ts)
           bucket.save(stats2, bucket.next_ts_bucket(ts))
@@ -105,10 +105,10 @@ module SimpleMetrics
 
       describe "#find_all_in_ts_by_name" do
         it "returns all stats for given name and timestamp" do
-          stats1a  =  Stats.create_counter(:name => "key1", :value => 5)
-          stats1b  =  Stats.create_counter(:name => "key1", :value => 7)
-          stats2   =  Stats.create_counter(:name => "key2", :value => 7)
-          stats1_different_ts   =  Stats.create_counter(:name => "key1", :value => 3)
+          stats1a  =  DataPoint.create_counter(:name => "key1", :value => 5)
+          stats1b  =  DataPoint.create_counter(:name => "key1", :value => 7)
+          stats2   =  DataPoint.create_counter(:name => "key2", :value => 7)
+          stats1_different_ts   =  DataPoint.create_counter(:name => "key1", :value => 3)
 
           bucket.save(stats1a, ts)
           bucket.save(stats1b, ts)
@@ -122,6 +122,51 @@ module SimpleMetrics
         end
       end
 
+      describe "#find_all_in_ts_by_wildcard" do
+        it "returns all stats for given name and timestamp" do
+          stats1  =  DataPoint.create_counter(:name => "com.test.key1", :value => 5)
+          stats2  =  DataPoint.create_counter(:name => "com.test.key2", :value => 7)
+          stats_different =  DataPoint.create_counter(:name => "com.test2.key1", :value => 3)
+
+          from = bucket.ts_bucket(ts)
+          to   = from
+          bucket.save(stats1, ts)
+          bucket.save(stats2, ts)
+          bucket.save(stats_different, ts)
+
+          results = bucket.find_all_in_ts_range_by_wildcard(from, to, "com.test.*")
+
+          results.should have(2).items
+          results.first.name.should == "com.test.key1"
+          results.last.name.should == "com.test.key2"
+        end
+      end
+
+      describe "#fill_gaps" do
+        it "returns stats and fills missing gaps with null entries" do
+          stats  =  DataPoint.create_counter(:name => "com.test.key1", :value => 5)
+
+          from = ts - 10
+          to   = ts + 10
+          bucket.save(stats, ts)
+          ts_bucket = bucket.ts_bucket(ts)
+
+          results = bucket.fill_gaps(from, to, [stats])
+
+          results.should have(3).items
+          results[0].name.should == "com.test.key1"
+          results[1].name.should == "com.test.key1"
+          results[2].name.should == "com.test.key1"
+
+          results[0].value.should be_nil
+          results[1].value.should == 5
+          results[2].value.should be_nil
+
+          results[0].ts.should == ts_bucket - 10
+          results[1].ts.should == ts_bucket
+          results[2].ts.should == ts_bucket + 10
+        end
+      end
     end # describe "finder methods"
 
     describe "#aggregate_all" do
@@ -130,10 +175,10 @@ module SimpleMetrics
         Mongo.ensure_collections_exist
       end
 
-      it "aggregates all stats" do
-        stats1a  =  Stats.create_counter(:name => "key1", :value => 5)
-        stats1b  =  Stats.create_counter(:name => "key1", :value => 7)
-        stats2   =  Stats.create_counter(:name => "key2", :value => 3)
+      it "aggregates all counter data points" do
+        stats1a  =  DataPoint.create_counter(:name => "key1", :value => 5)
+        stats1b  =  DataPoint.create_counter(:name => "key1", :value => 7)
+        stats2   =  DataPoint.create_counter(:name => "key2", :value => 3)
 
         bucket2 = Bucket[1]
         ts_at_insert = bucket2.previous_ts_bucket(ts)
@@ -154,28 +199,75 @@ module SimpleMetrics
         key2_result.value.should == 3
         key2_result.should be_counter
       end
+
+      it "aggregates all gauge data points" do
+        stats1a  =  DataPoint.create_gauge(:name => "key1", :value => 5)
+        stats1b  =  DataPoint.create_gauge(:name => "key1", :value => 7)
+        stats2   =  DataPoint.create_gauge(:name => "key2", :value => 3)
+
+        bucket2 = Bucket[1]
+        ts_at_insert = bucket2.previous_ts_bucket(ts)
+        bucket.save(stats1a, ts_at_insert)
+        bucket.save(stats1b, ts_at_insert)
+        bucket.save(stats2, ts_at_insert)
+
+        Bucket.aggregate_all(ts)
+
+        results = bucket2.find_all_in_ts(ts_at_insert)
+        results.should have(2).items
+
+        key1_result = results.find {|stat| stat.name == "key1"}
+        key1_result.value.should == 6
+        key1_result.should be_gauge
+
+        key2_result = results.find {|stat| stat.name == "key2"}
+        key2_result.value.should == 3
+        key2_result.should be_gauge
+      end
+
     end # describe "#aggregate_all"
 
-    describe "#flush_stats" do
+    describe "#flush_data_points" do
       before do
-        stats1 = Stats.create_counter(:name => "key1", :value => 5)
-        stats2 = Stats.create_counter(:name => "key1", :value => 7)
-        stats3 = Stats.create_counter(:name => "key2", :value => 3)
+        Mongo.truncate_collections
+        Mongo.ensure_collections_exist
+        
+        stats1 = DataPoint.create_counter(:name => "key1", :value => 5)
+        stats2 = DataPoint.create_counter(:name => "key1", :value => 7)
+        stats3 = DataPoint.create_counter(:name => "key2", :value => 3)
         @stats = [stats1, stats2, stats3]
       end
 
       it "saves all stats in finest/first bucket" do
-        Bucket.flush_stats(@stats)
+        Bucket.flush_data_points(@stats)
 
         results = bucket.find_all_in_ts(ts)
-        results.should have(3).items
+        results.should have(2).items
       end
 
       it "calls aggregate_all afterwards" do
         mock(Bucket).aggregate_all(ts)
-        Bucket.flush_stats(@stats)
+        Bucket.flush_data_points(@stats)
+      end
+
+      it "saves all stats and aggregate if duplicates found" do
+        Bucket.flush_data_points(@stats)
+
+        results = bucket.find_all_in_ts(ts)
+        results.should have(2).items
+        results.first.name.should == "key1"
+        results.last.name.should == "key2"
+        results.first.value == 12
+        results.last.value == 3
       end
-    end # describe "#flush_stats"
+
+      it "raises error if name matches but type does not" do
+        stats4 = DataPoint.create_gauge(:name => "key1", :value => 3)
+        input = @stats + [stats4]
+        expect { Bucket.flush_data_points(input) }.to raise_error(SimpleMetrics::DataPoint::NonMatchingTypesError)
+      end
+
+    end # describe "#flush_data_points"
 
   end
 end

data/spec/data_point_spec.rb

@@ -0,0 +1,131 @@
+# encoding: utf-8
+require "spec_helper"
+
+module SimpleMetrics
+
+  describe Bucket do
+
+    let(:ts) do
+      Time.now.utc.to_i
+    end
+
+    describe "#parse" do
+
+      it "parses increment counter" do
+        stats = DataPoint.parse("com.example.test1:1|c")
+        stats.name.should == "com.example.test1"
+        stats.value.should == 1
+        stats.should be_counter
+      end
+
+      it "parses decrement counter" do
+        stats = DataPoint.parse("com.example.test1:-1|c")
+        stats.name.should == "com.example.test1"
+        stats.value.should == -1
+        stats.should be_counter
+      end
+
+      it "parses counter with sample rate" do
+        stats = DataPoint.parse("com.example.test2:5|c|@0.1")
+        stats.name.should == "com.example.test2"
+        stats.value.should == 50
+        stats.should be_counter
+      end
+
+      it "parses increment gauge" do
+        stats = DataPoint.parse("com.example.test3:5|g")
+        stats.name.should == "com.example.test3"
+        stats.value.should == 5
+        stats.should be_gauge
+      end
+
+      it "parses increment gauge with sample rate" do
+        stats = DataPoint.parse("com.example.test3:5|g|@0.1")
+        stats.name.should == "com.example.test3"
+        stats.value.should == 50
+        stats.should be_gauge
+      end
+
+      it "parses increment timing" do
+        stats = DataPoint.parse("com.example.test4:44|ms")
+        stats.name.should == "com.example.test4"
+        stats.value.should == 44
+        stats.should be_timing
+      end
+
+      it "parses increment timing with sample rate" do
+      end
+    end
+
+
+    describe "#aggregate" do
+      it "aggregates counter data points" do
+        stats1  =  DataPoint.create_counter(:name => "key1", :value => 5)
+        stats2  =  DataPoint.create_counter(:name => "key1", :value => 7)
+        result = DataPoint.aggregate([stats1, stats2])
+        result.value.should == 12
+        result.name.should == "key1"
+        result.should be_counter
+      end
+
+      it "aggregates counter data points with custom name" do
+        stats1  =  DataPoint.create_counter(:name => "key1", :value => 5)
+        stats2  =  DataPoint.create_counter(:name => "key1", :value => 7)
+        result = DataPoint.aggregate([stats1, stats2], "new_name")
+        result.value.should == 12
+        result.name.should == "new_name"
+        result.should be_counter
+      end
+
+      it "aggregates gauge data points" do
+        stats1  =  DataPoint.create_gauge(:name => "key1", :value => 5)
+        stats2  =  DataPoint.create_gauge(:name => "key1", :value => 7)
+        result = DataPoint.aggregate([stats1, stats2])
+        result.value.should == 6
+        result.name.should == "key1"
+        result.should be_gauge
+      end
+
+      it "aggregates timing data points" do
+      end
+      
+      it "aggregates event data points" do
+      end
+    end
+
+     describe "#aggregate_array" do
+      it "aggregates counter data points" do
+        stats1  =  DataPoint.create_counter(:name => "com.test.key1", :value => 5, :ts => ts)
+        stats2  =  DataPoint.create_counter(:name => "com.test.key1", :value => 7, :ts => ts)
+        stats3  =  DataPoint.create_counter(:name => "com.test.key1", :value => 9, :ts => (ts + 60) )
+
+        results = DataPoint.aggregate_array([stats1, stats2, stats3], "com.test.*")
+        results.should have(2).data_points
+        results.first.name.should == "com.test.*"
+        results.last.name.should  == "com.test.*"
+        results.first.value.should == 12
+        results.last.value.should == 9
+      end
+
+      it "aggregates gauge data points" do
+        stats1  =  DataPoint.create_gauge(:name => "com.test.key1", :value => 5, :ts => ts)
+        stats2  =  DataPoint.create_gauge(:name => "com.test.key1", :value => 7, :ts => ts)
+        stats3  =  DataPoint.create_gauge(:name => "com.test.key1", :value => 9, :ts => (ts + 60) )
+
+        results = DataPoint.aggregate_array([stats1, stats2, stats3], "com.test.*")
+        results.should have(2).data_points
+        results.first.name.should == "com.test.*"
+        results.last.name.should  == "com.test.*"
+        results.first.value.should == 6
+        results.last.value.should == 9
+      end
+     
+      it "raises NonMatchingTypesError if types are different" do
+        stats1  =  DataPoint.create_counter(:name => "com.test.key1", :value => 5, :ts => ts)
+        stats2  =  DataPoint.create_gauge(:name => "com.test.key1", :value => 5, :ts => ts)
+        expect { DataPoint.aggregate_array([stats1, stats2], "com.test.*") }.to raise_error(SimpleMetrics::DataPoint::NonMatchingTypesError)
+      end
+    end
+  end
+
+end

data/spec/graph_spec.rb

@@ -0,0 +1,89 @@
+# encoding: utf-8
+require "spec_helper"
+
+module SimpleMetrics
+
+  describe Graph do
+
+  	let(:ts) do
+      Time.now.utc.to_i
+    end
+
+    let(:bucket) do
+      Bucket.first
+    end
+
+  	before do
+  	  Mongo.truncate_collections
+  	  Mongo.ensure_collections_exist
+  	end
+
+    describe "#query" do
+
+      it "returns string request data points as is" do
+      	dp1 = DataPoint.create_counter(:name => "key1", :value => 5)
+
+      	bucket.save(dp1, ts)
+
+        current_ts = bucket.ts_bucket(ts)
+      	from = current_ts
+      	to   = current_ts
+      	results = Graph.query(bucket, from, to, "key1")
+
+      	results.should have(1).data_point
+      end
+
+      it "returns string request data points and fill graps" do
+        dp1 = DataPoint.create_counter(:name => "key1", :value => 5)
+
+        bucket.save(dp1, ts)
+
+        current_ts = bucket.ts_bucket(ts)
+        from = current_ts
+        to   = current_ts+10
+        results = Graph.query(bucket, from, to, "key1")
+
+        results.should have(2).data_point
+        results.first.value.should == 5
+        results.last.value.should be_nil
+      end
+
+      it "returns wildcard request data points with aggregate counter" do
+      	dp1 = DataPoint.create_counter(:name => "com.test.key1", :value => 5)
+      	dp2 = DataPoint.create_counter(:name => "com.test.key2", :value => 7)
+
+      	bucket.save(dp1, ts)
+      	bucket.save(dp2, ts)
+
+      	current_ts = bucket.ts_bucket(ts)
+        from = current_ts
+        to   = current_ts
+      	results = Graph.query(bucket, from, to, "com.test.*")
+      	results.should have(1).data_point
+      	result = results.first
+      	result.name.should == "com.test.*"
+      	result.value.should == 12
+      	result.should be_counter
+      end
+
+      it "returns regexp request data points with aggregate gauge" do
+      	dp1 = DataPoint.create_gauge(:name => "com.test.key1", :value => 5)
+      	dp2 = DataPoint.create_gauge(:name => "com.test.key2", :value => 7)
+
+      	bucket.save(dp1, ts)
+      	bucket.save(dp2, ts)
+
+        current_ts = bucket.ts_bucket(ts)
+        from = current_ts
+        to   = current_ts
+
+      	results = Graph.query(bucket, from, to, /com\.test(.*)/)
+      	results.should have(1).data_point
+      	result = results.first
+      	result.name.should == "/com\\.test(.*)/"
+      	result.value.should == 6
+      	result.should be_gauge
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: simple_metrics
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.2.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-02 00:00:00.000000000 Z
+date: 2012-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70244480135940 !ruby/object:Gem::Requirement
+  requirement: &70101883011000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70244480135940
+  version_requirements: *70101883011000
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70244480135460 !ruby/object:Gem::Requirement
+  requirement: &70101883009820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70244480135460
+  version_requirements: *70101883009820
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &70244480135020 !ruby/object:Gem::Requirement
+  requirement: &70101883009000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70244480135020
+  version_requirements: *70101883009000
 - !ruby/object:Gem::Dependency
   name: eventmachine
-  requirement: &70244480134600 !ruby/object:Gem::Requirement
+  requirement: &70101883008200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70244480134600
+  version_requirements: *70101883008200
 - !ruby/object:Gem::Dependency
   name: daemons
-  requirement: &70244480134180 !ruby/object:Gem::Requirement
+  requirement: &70101883007560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,33 +65,34 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70244480134180
+  version_requirements: *70101883007560
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &70244480133740 !ruby/object:Gem::Requirement
+  requirement: &70101883006620 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.6'
   type: :runtime
   prerelease: false
-  version_requirements: *70244480133740
+  version_requirements: *70101883006620
 - !ruby/object:Gem::Dependency
-  name: bson_ext
-  requirement: &70244480133300 !ruby/object:Gem::Requirement
+  name: bson
+  requirement: &70101883005800 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.6'
   type: :runtime
   prerelease: false
-  version_requirements: *70244480133300
+  version_requirements: *70101883005800
 description: SimpleMetrics
 email:
 - fdietz@gmail.com
 executables:
+- populate
 - simple_metrics_client
 - simple_metrics_server
 extensions: []
@@ -102,20 +103,24 @@ files:
 - Gemfile
 - README.markdown
 - Rakefile
+- bin/populate
 - bin/simple_metrics_client
 - bin/simple_metrics_server
 - example/increment.rb
 - lib/simple_metrics.rb
 - lib/simple_metrics/bucket.rb
 - lib/simple_metrics/client.rb
+- lib/simple_metrics/data_point.rb
+- lib/simple_metrics/functions.rb
+- lib/simple_metrics/graph.rb
 - lib/simple_metrics/mongo.rb
 - lib/simple_metrics/server.rb
-- lib/simple_metrics/stats.rb
 - lib/simple_metrics/version.rb
 - simple_metrics.gemspec
 - spec/bucket_spec.rb
+- spec/data_point_spec.rb
+- spec/graph_spec.rb
 - spec/spec_helper.rb
-- spec/stats_spec.rb
 homepage: ''
 licenses: []
 post_install_message: 
@@ -128,19 +133,26 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 3607993347198601415
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 3607993347198601415
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.15
+rubygems_version: 1.8.17
 signing_key: 
 specification_version: 3
 summary: SimpleMetrics
 test_files:
 - spec/bucket_spec.rb
+- spec/data_point_spec.rb
+- spec/graph_spec.rb
 - spec/spec_helper.rb
-- spec/stats_spec.rb

data/lib/simple_metrics/stats.rb

@@ -1,99 +0,0 @@
-# encoding: utf-8
-module SimpleMetrics
-
-  class Stats
-
-    class NonMatchingTypesError < Exception; end
-    class ParserError           < Exception; end
-
-    # examples:
-    # com.example.test1:1|c
-    # com.example.test2:-1|c
-    # com.example.test2:50|g
-    # com.example.test3:5|c|@0.1
-    # com.example.test4:44|ms
-    REGEXP = /^([\d\w_.]*):(-?[\d]*)\|(c|g|ms){1}(\|@([.\d]+))?$/i
-
-    class << self
-
-      def parse(str)
-        if str =~ REGEXP
-          name, value, type, sample_rate = $1, $2, $3, $5
-          if type == "ms"
-            # TODO: implement sample_rate handling
-            create_timing(:name => name, :value => value)
-          elsif type == "g"
-            create_gauge(:name => name, :value => (value.to_i || 1) * (1.0 / (sample_rate || 1).to_f) )
-          elsif type == "c"
-            create_counter(:name => name, :value => (value.to_i || 1) * (1.0 / (sample_rate || 1).to_f) )
-          end
-        else
-          raise ParserError, "Parser Error - Invalid Stat: #{str}"
-        end
-      end
-
-      def create_counter(attributes)
-        Stats.new(attributes.merge(:type => 'c'))
-      end
-
-      def create_gauge(attributes)
-        Stats.new(attributes.merge(:type => 'g'))
-      end
-
-      def create_timing(attributes)
-        Stats.new(attributes.merge(:type => 'ms'))
-      end
-
-      def aggregate(stats_array)
-        raise NonMatchingTypesError unless stats_array.group_by { |stats| stats.type }.size == 1
-
-        result_stat = stats_array.first.dup
-        result_stat.value = stats_array.map { |stats| stats.value }.inject(0) { |result, value| result += value }
-        result_stat
-      end
-
-      def create_from_db(attributes)
-        Stats.new(:name => attributes["name"], :value => attributes["value"], :ts => attributes["ts"], :type => attributes["type"])
-      end
-    end
-
-    attr_accessor :name, :ts, :type, :value
-
-    def initialize(attributes)
-      @name  = attributes[:name]
-      @value = attributes[:value]
-      @ts    = attributes[:ts]
-      @type  = attributes[:type]
-    end
-
-    def counter?
-      type == 'c'
-    end
-
-    def gauge?
-      type == 'g'
-    end
-
-    def timing?
-      type == 'ms'
-    end
-
-    def timestamp
-      ts
-    end
-
-    def value
-      @value.to_i
-    end
-
-    def attributes
-      { 
-        :name  => name,
-        :value => value,
-        :ts    => ts,
-        :type  => type
-      }
-    end
-
-  end
-end

data/spec/stats_spec.rb

@@ -1,61 +0,0 @@
-# encoding: utf-8
-require "spec_helper"
-
-module SimpleMetrics
-
-  describe Bucket do
-
-    describe "#parse" do
-
-      it "parses increment counter" do
-        stats = Stats.parse("com.example.test1:1|c")
-        stats.name.should == "com.example.test1"
-        stats.value.should == 1
-        stats.should be_counter
-      end
-
-      it "parses decrement counter" do
-        stats = Stats.parse("com.example.test1:-1|c")
-        stats.name.should == "com.example.test1"
-        stats.value.should == -1
-        stats.should be_counter
-      end
-
-      it "parses counter with sample rate" do
-        stats = Stats.parse("com.example.test2:5|c|@0.1")
-        stats.name.should == "com.example.test2"
-        stats.value.should == 50
-        stats.should be_counter
-      end
-
-      it "parses increment gauge" do
-        stats = Stats.parse("com.example.test3:5|g")
-        stats.name.should == "com.example.test3"
-        stats.value.should == 5
-        stats.should be_gauge
-      end
-
-      it "parses increment gauge with sample rate" do
-        stats = Stats.parse("com.example.test3:5|g|@0.1")
-        stats.name.should == "com.example.test3"
-        stats.value.should == 50
-        stats.should be_gauge
-      end
-
-      it "parses increment timing" do
-        stats = Stats.parse("com.example.test4:44|ms")
-        stats.name.should == "com.example.test4"
-        stats.value.should == 44
-        stats.should be_timing
-      end
-
-      it "parses increment timing with sample rate" do
-      end
-    end
-
-
-    describe "create_counter" do
-    end
-  end
-
-end