simple_metrics 0.4.0 → 0.4.2

data/lib/simple_metrics/dashboard_repository.rb

@@ -0,0 +1,59 @@
+module SimpleMetrics
+  class DashboardRepository
+
+    class << self
+
+      def find_one(id)
+        dashboard(collection.find_one(BSON::ObjectId.from_string(id)))
+      end
+
+      def find_one_by_name(name)      
+        result = collection.find({ :name => name }).to_a.first
+        dashboard(result) if result
+      end
+
+      def find_all
+        results = collection.find.sort([['name', ::Mongo::ASCENDING]]).to_a
+        dashboards(results) if results
+      end
+
+      def save(dashboard)
+        collection.insert(dashboard.attributes.merge(:created_at => Time.now.utc, :updated_at => Time.now.utc))
+      end
+
+      def update(dashboard)
+        collection.update({ "_id" => dashboard.id }, { "$set" => dashboard.attributes.merge(:updated_at => Time.now.utc).reject { |k, v| k == 'id' } })  
+      end
+
+      def remove(id)
+        collection.remove("_id" => BSON::ObjectId.from_string(id))
+      end
+
+      def truncate_collections
+        collection.remove
+      end
+
+      def ensure_index
+        collection.ensure_index([['created_at', ::Mongo::ASCENDING]])
+        collection.ensure_index([['updated_at', ::Mongo::ASCENDING]])
+        collection.ensure_index([['name', ::Mongo::ASCENDING]])
+      end
+
+      private
+
+      def collection
+        Repository.db.collection('dashboards')
+      end
+
+      def dashboard(result)
+        Dashboard.new(:id => result["_id"], :name => result["name"], :instruments => result["instruments"], :created_at => result["created_at"], :updated_at => result["updated_at"])
+      end
+
+      def dashboards(results)
+        results.inject([]) { |result, a| result << dashboard(a); }
+      end
+
+    end # class << self
+
+  end
+end

data/lib/simple_metrics/data_point/base.rb

@@ -0,0 +1,59 @@
+# encoding: utf-8
+module SimpleMetrics
+  module DataPoint
+    class Base
+
+      attr_accessor :name, :ts, :type, :value, :total, :sum
+      attr_reader :id
+
+      def initialize(attributes)
+        @id          = attributes[:id]
+        @name        = attributes[:name]
+        @value       = attributes[:value]
+        @ts          = attributes[:ts]
+        @sample_rate = attributes[:sample_rate]
+        @sum         = attributes[:sum]
+        @total       = attributes[:total]
+      end
+
+      def counter?
+        @type == 'c'
+      end
+
+      def gauge?
+        @type == 'g'
+      end
+
+      def timing?
+        @type == 'ms'
+      end
+
+      def event?
+        @type == 'ev'
+      end
+
+      def timestamp
+        ts
+      end
+
+      def value
+        @value.to_i if @value
+      end
+
+      def attributes
+        { 
+          :name  => @name,
+          :value => @value,
+          :ts    => @ts,
+          :type  => @type,
+          :total => @total,
+          :sum   => @sum
+        }
+      end
+
+      def to_s
+        attributes.to_s
+      end
+    end
+  end
+end

data/lib/simple_metrics/data_point/counter.rb

@@ -0,0 +1,20 @@
+module SimpleMetrics
+  module DataPoint
+    class Counter < Base
+
+      def initialize(attributes)
+        super(attributes)
+        @type = 'c'
+        @value = (@value.to_i || 1) * (1.0 / (@sample_rate || 1).to_f)
+      end
+
+      def combine(dp)
+        @total += 1
+        @value += dp.value
+        @sum   += dp.value  
+        self
+      end
+
+    end
+  end
+end

data/lib/simple_metrics/data_point/event.rb

@@ -0,0 +1,16 @@
+module SimpleMetrics
+  module DataPoint
+    class Event < Base
+
+      def initialize(attributes)
+        super(attributes)
+        @type = 'ev'
+      end
+
+      def combine(dp)
+        raise "Implement me!"
+      end
+      
+    end
+  end
+end

data/lib/simple_metrics/data_point/gauge.rb

@@ -0,0 +1,19 @@
+module SimpleMetrics
+  module DataPoint
+    class Gauge < Base
+
+      def initialize(attributes)
+        super(attributes)
+        @type = 'g'
+        @value = (@value.to_i || 1) * (1.0 / (@sample_rate || 1).to_f)
+      end
+
+      def combine(dp)
+        @total += 1
+        @sum   += dp.value
+        @value  = @sum / @total 
+        self
+      end
+    end
+  end
+end

data/lib/simple_metrics/data_point/timing.rb

@@ -0,0 +1,15 @@
+module SimpleMetrics
+  module DataPoint
+    class Timing < Base
+
+      def initialize(attributes)
+        super(attributes)
+        @type = 'ms'
+      end
+
+      def combine(dp)
+        raise "Implement me!"
+      end
+    end
+  end
+end

data/lib/simple_metrics/data_point.rb

@@ -1,10 +1,10 @@
-# encoding: utf-8
 module SimpleMetrics
-
-  class DataPoint
+  module DataPoint
+    extend self
 
     class NonMatchingTypesError < Exception; end
     class ParserError           < Exception; end
+    class UnknownTypeError      < Exception; end
 
     # examples:
     # com.example.test1:1|c
@@ -14,149 +14,57 @@ module SimpleMetrics
     # com.example.test4:44|ms
     REGEXP = /^([\d\w_.]*):(-?[\d]*)\|(c|g|ms){1}(\|@([.\d]+))?$/i
 
-    class << self
-
-      def parse(str)
-        if str =~ REGEXP
-          name, value, type, sample_rate = $1, $2, $3, $5
-          if type == "ms"
-            # TODO: implement sample_rate handling
-            create_timing(:name => name, :value => value)
-          elsif type == "g"
-            create_gauge(:name => name, :value => (value.to_i || 1) * (1.0 / (sample_rate || 1).to_f) )
-          elsif type == "c"
-            create_counter(:name => name, :value => (value.to_i || 1) * (1.0 / (sample_rate || 1).to_f) )
-          end
-        else
-          raise ParserError, "Parser Error - Invalid Stat: #{str}"
-        end
-      end
-
-      def create_counter(attributes)
-        self.new(attributes.merge(:type => 'c'))
-      end
-
-      def create_gauge(attributes)
-        self.new(attributes.merge(:type => 'g'))
-      end
-
-      def create_timing(attributes)
-        self.new(attributes.merge(:type => 'ms'))
-      end
-
-      def aggregate(stats_array, name = nil)
-        raise NonMatchingTypesError unless stats_array.group_by { |stats| stats.type }.size == 1
-
-        result_stat      = stats_array.first.dup
-        result_stat.name = name if name
-        if stats_array.first.counter?
-          result_stat.value = stats_array.map { |stats| stats.value }.inject(0) { |result, value| result += value }
-          result_stat
-        elsif stats_array.first.gauge?
-          total_value = stats_array.map { |stats| stats.value }.inject(0) { |result, value| result += value }
-          result_stat.value = total_value / stats_array.size
-          result_stat
-        elsif stats_array.first.timing?
-          # TODO implement timing aggregation
-        elsif stats_array.first.event? 
-          # TODO implement event aggregation
-        else
-          raise ArgumentError, "Unknown data point type"
-        end
-      end
-
-      def aggregate_array(stats_array, name = nil)
-        raise NonMatchingTypesError unless stats_array.group_by { |stats| stats.type }.size == 1
-
-        if stats_array.first.counter?
-          tmp_hash = ts_hash_aggregated(stats_array) do |value1, value2|
-            value1 + value2
-          end
-
-          result = []
-          tmp_hash.each_pair do |key, value|
-            result << self.new(:name => name, :ts => key, :value => value, :type => 'c')
-          end
-          result
-        elsif stats_array.first.gauge?
-          tmp_hash = ts_hash_aggregated(stats_array) do |value1, value2|
-            (value1 + value2)/2
-          end
-
-          result = []
-          tmp_hash.each_pair do |key, value|
-            result << self.new(:name => name, :ts => key, :value => value, :type => 'g')
-          end
-          result
-        elsif stats_array.first.timing?
-          # TODO implement timing aggregation
-        elsif stats_array.first.event? 
-          # TODO implement event aggregation
-        else
-          raise ArgumentError, "Unknown data point type"
-        end
-      end
-
-      def create_from_db(attributes)
-        self.new(:name => attributes["name"], :value => attributes["value"], :ts => attributes["ts"], :type => attributes["type"])
-      end
-
-      def ts_hash(query_result)
-        query_result.inject({}) { |result, dp| result[dp.ts] = dp; result }
-      end
-      
-      private
-
-      def ts_hash_aggregated(data_points, &block)
-        tmp = {}
-        data_points.each do |dp|
-          if tmp.key?(dp.ts)
-            tmp[dp.ts] = block.call(tmp[dp.ts], dp.value)
-          else
-            tmp[dp.ts] = dp.value
-          end
-        end
-        tmp
+    def parse(str)
+      if str =~ REGEXP
+        name, value, type, sample_rate = $1, $2, $3, $5
+        build(:name => name, :value => value, :type => type, :sample_rate => sample_rate)
+      else
+        raise ParserError, "Parser Error - Invalid data point: #{str}"
       end
     end
 
-    attr_accessor :name, :ts, :type, :value
-
-    def initialize(attributes)
-      @name  = attributes[:name]
-      @value = attributes[:value]
-      @ts    = attributes[:ts]
-      @type  = attributes[:type]
-    end
-
-    def counter?
-      type == 'c'
-    end
-
-    def gauge?
-      type == 'g'
+    def build(attributes)
+      case attributes[:type]
+      when 'c'
+        Counter.new(attributes)
+      when 'g'
+        Gauge.new(attributes) 
+      when 'ms'
+        Timing.new(attributes)
+      when 'ev'
+        Event.new(attributes)
+      else
+        raise UnknownTypeError, "Unknown Type Error: #{attributes[:type]}"
+      end
     end
-
-    def timing?
-      type == 'ms'
+    
+    def aggregate_values(dps)
+      raise SimpleMetrics::DataPoint::NonMatchingTypesError if has_non_matching_types?(dps)
+
+      dp       = dps.first.dup
+      dp.value = if dp.counter?
+        sum(dps)
+      elsif dp.gauge?
+        sum(dps) / dps.size
+      elsif dp.event?
+        raise "Implement me!"
+      elsif dp.timing?
+        raise "Implement me!"
+      else
+        raise ArgumentError("Unknown data point type: #{dp}")
+      end
+      dp
     end
 
-    def timestamp
-      ts
-    end
+    private
 
-    def value
-      @value.to_i if @value
+    def sum(dps)
+      dps.map { |dp| dp.value }.inject(0) { |result, value| result += value }
     end
 
-    def attributes
-      { 
-        :name  => name,
-        :value => value,
-        :ts    => ts,
-        :type  => type
-      }
+    def has_non_matching_types?(dps)
+      dps.group_by { |dp| dp.type }.size != 1
     end
 
   end
-end
+end

data/lib/simple_metrics/data_point_repository.rb

@@ -0,0 +1,115 @@
+require "mongo"
+
+module SimpleMetrics
+  class DataPointRepository
+
+    class << self
+
+      @@collection = {}
+
+      def for_retention(name)
+        self.new(collection(name))
+      end
+
+      def collection(name)
+        raise ArgumentError, "Unknown retention: #{name}" unless retention_names.include?(name)
+        @@collection[name] ||= db.collection(name)
+      end
+
+      def ensure_collections_exist
+        SimpleMetrics.logger.debug "SERVER: MongoDB - found following collections: #{db.collection_names.inspect}"
+        buckets.each do |retention|
+          unless db.collection_names.include?(retention.fetch(:name))
+            db.create_collection(retention.fetch(:name), :capped => retention.fetch(:capped), :size => retention.fetch(:size)) 
+            SimpleMetrics.logger.debug "SERVER: MongoDB - created collection #{retention.fetch(:name)}, capped: #{retention.fetch(:capped)}, size: #{retention.fetch(:size)}"
+          end
+          
+          db.collection(retention.fetch(:name)).ensure_index([['ts', ::Mongo::ASCENDING]])
+          db.collection(retention.fetch(:name)).ensure_index([['_id', ::Mongo::ASCENDING]])
+          SimpleMetrics.logger.debug "SERVER: MongoDB - ensure index on column ts for collection #{retention.fetch(:name)}"
+        end 
+      end
+
+      def truncate_collections
+        buckets.each do |retention|
+          if db.collection_names.include?(retention.fetch(:name))
+            if retention.fetch(:capped)
+              collection(retention.fetch(:name)).drop # capped collections can't remove elements, drop it instead
+            else
+              collection(retention.fetch(:name)).remove
+            end
+            SimpleMetrics.logger.debug "SERVER: MongoDB - truncated collection #{retention.fetch(:name)}"
+          end
+        end
+      end
+
+      private
+
+      def db
+        Repository.db
+      end
+
+      def retention_names
+        buckets.map { |r| r.fetch(:name) }
+      end
+
+      def buckets
+        SimpleMetrics.config.buckets
+      end
+    end
+
+    def initialize(collection)
+      @collection = collection
+    end
+
+    def save(result)
+      @collection.insert(result.attributes.reject { |k, v| k == 'id' })
+    end
+
+    def update(dp, ts)
+      @collection.update({ "_id" => dp.id }, { "$set" => { :value => dp.value, :sum => dp.sum, :total => dp.total }})  
+    end
+
+    def find_all_at_ts(ts)
+      results = @collection.find({ :ts => ts }).to_a
+      data_points(results)
+    end
+
+    def find_data_point_at_ts(ts, name)
+      result = @collection.find_one({ :ts => ts, :name => name })
+      data_point(result) if result
+    end
+
+    def find_all_in_ts_range_by_name(from, to, name)
+      results = @collection.find({ :name => name }.merge(range_query(from, to))).to_a
+      data_points(results)
+    end
+
+    def find_all_in_ts_range_by_wildcard(from, to, target)
+      results = @collection.find({ :name => regexp(target) }.merge(range_query(from, to))).to_a
+      data_points(results)
+    end
+
+    private
+
+    def regexp(target)
+      /#{wildcard_replace(target)}/
+    end
+
+    def wildcard_replace(target)
+      target.gsub('.', '\.').gsub('*', '.*')
+    end
+
+    def range_query(from, to)
+      { :ts => { "$gte" => from, "$lte" => to } }
+    end
+
+    def data_point(result)
+      DataPoint.build(:id => result["_id"], :name => result["name"], :value => result["value"], :ts => result["ts"], :type => result["type"], :sum => result["sum"], :total => result["total"])
+    end
+
+    def data_points(results)
+      results.inject([]) { |result, a| result << data_point(a); }
+    end
+  end
+end

data/lib/simple_metrics/functions.rb

@@ -2,14 +2,14 @@ module SimpleMetrics
 
 	module Functions
 
-		# calculate the maximum value for multiple targets
-		#
-		# params:
+  	# calculate the maximum value for multiple targets
+  	#
+  	# params:
     #  data_points   [1, 3, 5], [2, 1, 6]
     #
     # return:
     #  array         [2, 3, 6]
-		def max(*data_points)
+	 def max(*data_points)
     end
 
     # calculate the minimum value for multiple targets
@@ -17,7 +17,7 @@ module SimpleMetrics
     end
 
     # add offset to each value
-		def offset(*data_points)
+	 def offset(*data_points)
     end
 
     # multiple each value 

data/lib/simple_metrics/graph.rb

@@ -1,58 +1,95 @@
 module SimpleMetrics
 
-  # 
-  # url format examples:
-  # * target=com.post.clicks (1 line in graph)
-  # * target=com.post.clicks.text&target=com.post.clicks.logo (2 lines in graph)
-  # * target=com.post.clicks.* (1 aggregated line in graph)
-  #
   module Graph
     extend self
 
-    def minutes
-      Bucket[0]
-    end
-
-    def hours
-      Bucket[1]
-    end
+    def minutes; Bucket[0]; end
+    def hours;   Bucket[1]; end
+    def day;     Bucket[2]; end
+    def week;    Bucket[3]; end
 
-    def day
-      Bucket[2]
-    end
-
-    def week
-      Bucket[3]
+    def time_range(time)
+      case time
+      when 'minute'
+        5 * one_minute
+      when 'hour'
+        one_hour
+      when 'day'
+        one_day
+      when 'week'
+        one_week
+      else 
+        raise "Unknown time param: #{time}"
+      end
     end
-
+    
     def query_all(bucket, from, to, *targets)
-      result = {}
+      results = []
       Array(targets).each do |target|
-        result[target.inspect] = values_only(query(bucket, from, to, target))
+        results << { :name => target, :data => query(bucket, from, to, target).map { |data| { :x => data.ts, :y => data.value || 0 } } }
       end
-      result
+      results
     end
 
     def query(bucket, from, to, target)
-      if target.is_a?(Regexp) 
-        result = bucket.find_all_in_ts_range_by_regexp(from, to, target)
-        result = DataPoint.aggregate_array(result, target.inspect)
-        bucket.fill_gaps(from, to, result)
-      elsif target.is_a?(String) && target.include?('*')
+      if wild_card_query?(target)
         result = bucket.find_all_in_ts_range_by_wildcard(from, to, target)
-        result = DataPoint.aggregate_array(result, target)
+        result = aggregate(result, target)
         bucket.fill_gaps(from, to, result)
       elsif target.is_a?(String)
         result = bucket.find_all_in_ts_range_by_name(from, to, target)
         bucket.fill_gaps(from, to, result) 
       else
-        raise ArgumentError, "Unknown target: #{target.inspect}"
+        raise ArgumentError, "Unknown target format: #{target.inspect}"
+      end
+    end
+
+    private 
+
+    def aggregate(dps, target)
+      raise SimpleMetrics::DataPoint::NonMatchingTypesError if has_non_matching_types?(dps)
+
+      tmp = {}
+      dps.each do |dp|
+        dp.name = target
+        if tmp.key?(dp.ts)
+          tmp[dp.ts] << dp
+        else
+          tmp[dp.ts] = [dp]
+        end
+      end
+      tmp
+
+      result = []
+      tmp.each_pair do |ts, dps|
+        result << DataPoint.aggregate_values(dps)
       end
+      result
     end
 
-    def values_only(data_point_array)
-      data_point_array.map { |data| { :ts => data.ts, :value => data.value } }
+    def has_non_matching_types?(dps)
+      dps.group_by { |dp| dp.type }.size != 1
     end
 
+    def one_minute
+      60
+    end
+
+    def one_hour
+      one_minute * 60
+    end
+
+    def one_day
+      one_hour * 24
+    end
+
+    def one_week
+      one_day * 7
+    end
+
+    def wild_card_query?(target)
+      target.is_a?(String) && target.include?('*')
+    end
+  
   end
 end