simple_metrics 0.2.3 → 0.3.2

data/lib/simple_metrics/bucket.rb

@@ -5,7 +5,7 @@ module SimpleMetrics
     class << self
 
       def all
-        @@all ||= SimpleMetrics.buckets_config.map { |r| Bucket.new(r) }
+        @@all ||= SimpleMetrics.config.buckets.map { |r| Bucket.new(r) }
       end
 
       def first
@@ -17,44 +17,62 @@ module SimpleMetrics
         all[index]
       end
 
-      def coarse_buckets
-        Bucket.all.sort_by! { |r| r.seconds }[1..-1]
+      def flush_raw_data(data)      
+        data_points = []
+        data.each do |str|
+          begin
+            data_points << DataPoint.parse(str)
+          rescue DataPoint::ParserError => e
+            SimpleMetrics.logger.debug "Invalid Data skipped: #{str}, #{e}"
+          end
+        end
+        flush_data_points(data_points, Time.now.utc.to_i)
       end
 
-      def flush_data_points(data_points)
+      def flush_data_points(data_points, ts = nil)
         return if data_points.empty?
         SimpleMetrics.logger.info "#{Time.now} Flushing #{data_points.count} counters to MongoDB"
 
-        ts = Time.now.utc.to_i
+        ts   ||= Time.now.utc.to_i
         bucket = Bucket.first
-        data_points.group_by { |data| data.name }.each_pair do |name,dps|
-          data = DataPoint.aggregate(dps)
-          bucket.save(data, ts)
-        end
 
-        self.aggregate_all(ts)
+        data_points.group_by { |dp| dp.name }.each_pair do |name,dps|
+          dp = ValueAggregation.aggregate(dps)
+          bucket.save(dp, ts)
+          update_metric(dp)
+          aggregate(dp)  
+        end
       end
 
-      def aggregate_all(ts)
-        ts_bucket = self.first.ts_bucket(ts)
-
+      def aggregate(dp)
         coarse_buckets.each do |bucket|
-          current_ts = bucket.ts_bucket(ts_bucket)
-          previous_ts = bucket.previous_ts_bucket(ts_bucket)
-          SimpleMetrics.logger.debug "Aggregating #{bucket.name} #{previous_ts}....#{current_ts} (#{humanized_timestamp(previous_ts)}..#{humanized_timestamp(current_ts)})"
-
-          unless bucket.stats_exist_in_previous_ts?(previous_ts)
-            data_points = self.first.find_all_in_ts_range(previous_ts, current_ts)
-            data_points.group_by { |data| data.name }.each_pair do |name,dps|
-              data = DataPoint.aggregate(dps)
-              bucket.save(data, previous_ts)
-            end
+          existing_dp = bucket.find_data_point_at_ts(dp.ts, dp.name)
+          if existing_dp
+            UpdateAggregation.aggregate(existing_dp, dp)
+            bucket.update(existing_dp, existing_dp.ts)
+          else
+            dp.sum   = dp.value
+            dp.total = 1
+            bucket.save(dp, dp.ts)
           end
         end
       end
 
       private
 
+      def update_metric(dp)
+        metric = MetricRepository.find_one_by_name(dp.name)
+        if metric
+          MetricRepository.update(Metric.new(:name => dp.name, :total => metric.total + 1))
+        else
+          MetricRepository.save(Metric.new(:name => dp.name, :total => 1))
+        end
+      end
+
+      def coarse_buckets
+        Bucket.all.sort_by! { |r| r.seconds }[1..-1]
+      end
+
       def humanized_timestamp(ts)
         Time.at(ts).utc
       end
@@ -63,10 +81,10 @@ module SimpleMetrics
     attr_reader :name, :capped
 
     def initialize(attributes)
-      @name    = attributes[:name]
-      @seconds = attributes[:seconds]
-      @capped  = attributes[:capped]
-      @size    = attributes[:size]
+      @name    = attributes['name']
+      @seconds = attributes['seconds']
+      @capped  = attributes['capped']
+      @size    = attributes['size']
     end
 
     def seconds
@@ -78,7 +96,7 @@ module SimpleMetrics
     end
 
     def ts_bucket(ts)
-      ts / seconds * seconds
+      (ts / seconds) * seconds
     end
 
     def next_ts_bucket(ts)
@@ -89,70 +107,59 @@ module SimpleMetrics
       ts_bucket(ts) - seconds
     end
 
-    def find(id)
-      mongo_result = mongo_coll.find_one({ :_id => id })
-      DataPoint.create_from_db(mongo_result)
-    end
-
-    def find_all_by_name(name)
-      mongo_result = mongo_coll.find({ :name => name }).to_a
-      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
-    end
-
-    def find_all_in_ts(ts)
-      mongo_result = mongo_coll.find({ :ts => ts_bucket(ts) }).to_a
-      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
+    # TODO: only used in tests, do we need it?
+    def find_all_at_ts(ts)
+      repository.find_all_at_ts(ts_bucket(ts))
     end
 
-    def find_all_in_ts_by_name(ts, name)
-      mongo_result = mongo_coll.find({ :ts => ts_bucket(ts), :name => name }).to_a
-      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
+    def find_data_point_at_ts(ts, name)
+      repository.find_data_point_at_ts(ts_bucket(ts), name)
     end
 
+    # TODO: only used in tests, do we need it?
     def find_all_in_ts_range(from, to)
-      mongo_result = mongo_coll.find({ :ts => { "$gte" => from, "$lte" => to }}).to_a
-      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
+      repository.find_all_in_ts_range(from, to)
     end
 
     def find_all_in_ts_range_by_name(from, to, name)
-      mongo_result = mongo_coll.find({ :name => name, :ts => { "$gte" => from, "$lte" => to }}).to_a
-      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
+      repository.find_all_in_ts_range_by_name(from, to, name)
     end
 
     def find_all_in_ts_range_by_wildcard(from, to, target)
-      target = target.gsub('.', '\.')
-      target = target.gsub('*', '.*')
-      mongo_result = mongo_coll.find({ :name => /#{target}/, :ts => { "$gte" => from, "$lte" => to } }).to_a
-      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
+      repository.find_all_in_ts_range_by_wildcard(from, to, target)
     end
 
-    def find_all_in_ts_range_by_regexp(from, to, target)
-      mongo_result = mongo_coll.find({ :name => /#{target}/, :ts => { "$gte" => from, "$lte" => to } }).to_a
-      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
+    # TODO: only used in tests, do we need it?
+    def data_points_exist_at_ts?(ts, name)
+      repository.count_for_name_at(ts, name) > 0
     end
 
+    # TODO: only used in tests, do we need it?
     def stats_exist_in_previous_ts?(ts)
-      mongo_coll.find({ :ts => ts }).count > 0
+      repository.count_at(ts) > 0
     end
 
     def find_all_distinct_names
-      mongo_coll.distinct(:name).to_a
+      repository.find_all_distinct_names
     end
 
-    def save(stats, ts)
-      stats.ts = ts_bucket(ts)
-      result = mongo_coll.insert(stats.attributes)
-      SimpleMetrics.logger.debug "SERVER: MongoDB - insert in #{name}: #{stats.inspect}, result: #{result}"
+    def save(dp, ts)
+      dp.ts = ts_bucket(ts)
+      repository.save(dp)
+      SimpleMetrics.logger.debug "SERVER: MongoDB - insert in #{name}: #{dp.inspect}"
     end
 
-    def mongo_coll
-      Mongo.collection(name)
+    def update(dp, ts)
+      dp.ts = ts_bucket(ts)
+      repository.update(dp, ts)
+      SimpleMetrics.logger.debug "SERVER: MongoDB - update in #{name}: #{dp.inspect}"
     end
 
     def capped?
       @capped == true
     end
 
+    # TODO refactor, move to graph.rb
     def fill_gaps(from, to, query_result)
       return query_result if query_result.nil? || query_result.size == 0
       
@@ -176,6 +183,10 @@ module SimpleMetrics
 
     private
 
+    def repository
+      DataPointRepository.for_retention(name)
+    end
+
     def each_ts(from, to)
       current_bucket_ts = ts_bucket(from)
       while (current_bucket_ts <= ts_bucket(to))

data/lib/simple_metrics/configuration.rb

@@ -0,0 +1,79 @@
+require 'yaml'
+
+module SimpleMetrics
+  class Configuration
+
+    attr_reader :config
+
+    def initialize(hash = {}, &block)
+      @config = load_defaults.merge(hash)  
+    end
+
+    def configure(hash = {}, &block)
+      yield self if block_given?
+      self
+    end
+
+    def db
+      @db ||= config['db']
+    end
+
+    def db=(db)
+      @db = db
+    end
+
+    def buckets
+      @buckets ||= config['buckets']
+    end
+
+    def buckets=(buckets)
+      @buckets = buckets
+    end
+
+    def server
+      @server ||= config['server']
+    end
+
+    def server=(server)
+      @server = server
+    end
+
+    def web
+      @web ||= config['web']
+    end
+
+    def web=(web)
+      @web = web
+    end
+
+    private
+
+    def load_defaults
+      @config = load_config
+    rescue Errno::ENOENT # not found error
+      logger.info "Creating initial config file: #{config_file}"
+      FileUtils.cp(default_config_file, config_file)
+      @config = load_config
+    end
+
+    def config_file
+      File.expand_path('~/.simple_metrics.conf')
+    end
+
+    def default_config_file
+      File.expand_path('../../../default_config.yml', __FILE__)
+    end
+
+    def load_config
+      YAML.load_file(config_file)  
+    rescue ArgumentError => e
+      logger.error "Error parsing config file: #{e}"
+    rescue IOError => e
+      logger.error "Error reading config file: #{e}"
+    end
+
+    def logger
+      SimpleMetrics.logger
+    end
+  end
+end

data/lib/simple_metrics/data_point/base.rb

@@ -0,0 +1,59 @@
+# encoding: utf-8
+module SimpleMetrics
+  module DataPoint
+    class Base
+
+      attr_accessor :name, :ts, :type, :value, :total, :sum
+      attr_reader :id
+
+      def initialize(attributes)
+        @id          = attributes[:id]
+        @name        = attributes[:name]
+        @value       = attributes[:value]
+        @ts          = attributes[:ts]
+        @sample_rate = attributes[:sample_rate]
+        @sum         = attributes[:sum]
+        @total       = attributes[:total]
+      end
+
+      def counter?
+        @type == 'c'
+      end
+
+      def gauge?
+        @type == 'g'
+      end
+
+      def timing?
+        @type == 'ms'
+      end
+
+      def event?
+        @type == 'ev'
+      end
+
+      def timestamp
+        ts
+      end
+
+      def value
+        @value.to_i if @value
+      end
+
+      def attributes
+        { 
+          :name  => @name,
+          :value => @value,
+          :ts    => @ts,
+          :type  => @type,
+          :total => @total,
+          :sum   => @sum
+        }
+      end
+
+      def to_s
+        attributes.to_s
+      end
+    end
+  end
+end

data/lib/simple_metrics/data_point/counter.rb

@@ -0,0 +1,13 @@
+module SimpleMetrics
+  module DataPoint
+    class Counter < Base
+
+      def initialize(attributes)
+        super(attributes)
+        @type = 'c'
+        @value = (@value.to_i || 1) * (1.0 / (@sample_rate || 1).to_f)
+      end
+
+    end
+  end
+end

data/lib/simple_metrics/data_point/event.rb

@@ -0,0 +1,12 @@
+module SimpleMetrics
+  module DataPoint
+    class Event < Base
+
+      def initialize(attributes)
+        super(attributes)
+        @type = 'ev'
+      end
+
+    end
+  end
+end

data/lib/simple_metrics/data_point/gauge.rb

@@ -0,0 +1,13 @@
+module SimpleMetrics
+  module DataPoint
+    class Gauge < Base
+
+      def initialize(attributes)
+        super(attributes)
+        @type = 'g'
+        @value = (@value.to_i || 1) * (1.0 / (@sample_rate || 1).to_f)
+      end
+
+    end
+  end
+end

data/lib/simple_metrics/data_point/timing.rb

@@ -0,0 +1,12 @@
+module SimpleMetrics
+  module DataPoint
+    class Timing < Base
+
+      def initialize(attributes)
+        super(attributes)
+        @type = 'ms'
+      end
+
+    end
+  end
+end

data/lib/simple_metrics/data_point.rb

@@ -1,10 +1,10 @@
-# encoding: utf-8
 module SimpleMetrics
-
-  class DataPoint
+  module DataPoint
+    extend self
 
     class NonMatchingTypesError < Exception; end
     class ParserError           < Exception; end
+    class UnknownTypeError      < Exception; end
 
     # examples:
     # com.example.test1:1|c
@@ -14,149 +14,45 @@ module SimpleMetrics
     # com.example.test4:44|ms
     REGEXP = /^([\d\w_.]*):(-?[\d]*)\|(c|g|ms){1}(\|@([.\d]+))?$/i
 
-    class << self
-
-      def parse(str)
-        if str =~ REGEXP
-          name, value, type, sample_rate = $1, $2, $3, $5
-          if type == "ms"
-            # TODO: implement sample_rate handling
-            create_timing(:name => name, :value => value)
-          elsif type == "g"
-            create_gauge(:name => name, :value => (value.to_i || 1) * (1.0 / (sample_rate || 1).to_f) )
-          elsif type == "c"
-            create_counter(:name => name, :value => (value.to_i || 1) * (1.0 / (sample_rate || 1).to_f) )
-          end
-        else
-          raise ParserError, "Parser Error - Invalid Stat: #{str}"
-        end
-      end
-
-      def create_counter(attributes)
-        self.new(attributes.merge(:type => 'c'))
-      end
-
-      def create_gauge(attributes)
-        self.new(attributes.merge(:type => 'g'))
-      end
-
-      def create_timing(attributes)
-        self.new(attributes.merge(:type => 'ms'))
-      end
-
-      def aggregate(stats_array, name = nil)
-        raise NonMatchingTypesError unless stats_array.group_by { |stats| stats.type }.size == 1
-
-        result_stat      = stats_array.first.dup
-        result_stat.name = name if name
-        if stats_array.first.counter?
-          result_stat.value = stats_array.map { |stats| stats.value }.inject(0) { |result, value| result += value }
-          result_stat
-        elsif stats_array.first.gauge?
-          total_value = stats_array.map { |stats| stats.value }.inject(0) { |result, value| result += value }
-          result_stat.value = total_value / stats_array.size
-          result_stat
-        elsif stats_array.first.timing?
-          # TODO implement timing aggregation
-        elsif stats_array.first.event? 
-          # TODO implement event aggregation
-        else
-          raise ArgumentError, "Unknown data point type"
-        end
-      end
-
-      def aggregate_array(stats_array, name = nil)
-        raise NonMatchingTypesError unless stats_array.group_by { |stats| stats.type }.size == 1
-
-        if stats_array.first.counter?
-          tmp_hash = ts_hash_aggregated(stats_array) do |value1, value2|
-            value1 + value2
-          end
-
-          result = []
-          tmp_hash.each_pair do |key, value|
-            result << self.new(:name => name, :ts => key, :value => value, :type => 'c')
-          end
-          result
-        elsif stats_array.first.gauge?
-          tmp_hash = ts_hash_aggregated(stats_array) do |value1, value2|
-            (value1 + value2)/2
-          end
-
-          result = []
-          tmp_hash.each_pair do |key, value|
-            result << self.new(:name => name, :ts => key, :value => value, :type => 'g')
-          end
-          result
-        elsif stats_array.first.timing?
-          # TODO implement timing aggregation
-        elsif stats_array.first.event? 
-          # TODO implement event aggregation
-        else
-          raise ArgumentError, "Unknown data point type"
-        end
-      end
-
-      def create_from_db(attributes)
-        self.new(:name => attributes["name"], :value => attributes["value"], :ts => attributes["ts"], :type => attributes["type"])
-      end
-
-      def ts_hash(query_result)
-        query_result.inject({}) { |result, dp| result[dp.ts] = dp; result }
-      end
-      
-      private
-
-      def ts_hash_aggregated(data_points, &block)
-        tmp = {}
-        data_points.each do |dp|
-          if tmp.key?(dp.ts)
-            tmp[dp.ts] = block.call(tmp[dp.ts], dp.value)
-          else
-            tmp[dp.ts] = dp.value
-          end
-        end
-        tmp
+    def parse(str)
+      if str =~ REGEXP
+        name, value, type, sample_rate = $1, $2, $3, $5
+        build(:name => name, :value => value, :type => type, :sample_rate => sample_rate)
+      else
+        raise ParserError, "Parser Error - Invalid data point: #{str}"
       end
     end
 
-    attr_accessor :name, :ts, :type, :value
-
-    def initialize(attributes)
-      @name  = attributes[:name]
-      @value = attributes[:value]
-      @ts    = attributes[:ts]
-      @type  = attributes[:type]
-    end
-
-    def counter?
-      type == 'c'
-    end
-
-    def gauge?
-      type == 'g'
+    def build(attributes)
+      case attributes[:type]
+      when 'c'
+        DataPoint.create_counter(attributes)
+      when 'g'
+        Gauge.new(attributes) 
+      when 'ms'
+        Timing.new(attributes)
+      when 'ev'
+        Event.new(attributes)
+      else
+        raise UnknownTypeError, "Unknown Type Error: #{attributes[:type]}"
+      end
     end
 
-    def timing?
-      type == 'ms'
+    def create_counter(attributes)
+      Counter.new(attributes)
     end
 
-    def timestamp
-      ts
+    def create_gauge(attributes)
+      Gauge.new(attributes) 
     end
 
-    def value
-      @value.to_i if @value
+    def create_timing(attributes)
+      Timing.new(attributes)
     end
 
-    def attributes
-      { 
-        :name  => name,
-        :value => value,
-        :ts    => ts,
-        :type  => type
-      }
+    def ts_hash(query_result)
+      query_result.inject({}) { |result, dp| result[dp.ts] = dp; result }
     end
-
+    
   end
-end
+end