simple_metrics 0.3.2 → 0.3.3

data/bin/populate

@@ -21,7 +21,7 @@ def create_dps(name)
   previous_value = 20
   (1..1).inject([]) do |result, index|
     value = previous_value+rand(20)
-    result << SimpleMetrics::DataPoint.create_counter(:name => name, :value => value)
+    result << SimpleMetrics::DataPoint::Counter.new(:name => name, :value => value)
     previous_value = value
     result
   end
@@ -32,7 +32,7 @@ current = now - 1 * hour
 while (current < now)
   dps = create_dps(name)
   puts "flush data for #{Time.at(current)}, #{counter}"
-  SimpleMetrics::Bucket.flush_data_points(dps, current)
+  SimpleMetrics::Importer.flush_data_points(dps, current)
   current += 10
   counter += 1
 end

data/lib/simple_metrics/app.rb

@@ -13,18 +13,6 @@ module SimpleMetrics
     set :public_folder, File.expand_path('../public', __FILE__)
 
     helpers do
-      def graph_title(time)
-        case time
-        when "minute"
-          "Minute"
-        when "hour"
-          "Hour"
-        when "day"
-          "Day"
-        when "week"
-          "Week"
-        end
-      end
     end
 
     get "/api/metrics" do
@@ -33,9 +21,9 @@ module SimpleMetrics
       metrics.inject([]) { |result, m| result << m.attributes }.to_json
     end
 
-    get "/api/metrics/:id" do
+    get "/api/metrics/:name" do
       content_type :json
-      metric = SimpleMetrics::MetricRepository.find_one(to_bson_id(params[:id]))
+      metric = SimpleMetrics::MetricRepository.find_one_by_name(params[:name])
       metric.attributes.to_json
     end
 
@@ -52,85 +40,13 @@ module SimpleMetrics
       erb :index
     end
 
-    # get "/metrics" do
-    #   erb :index
-    # end
-
-    # get "/metrics/:id" do
-    #   erb :index
-    # end
-
-    # get "/metric" do
-    #   @from = (params[:from]  || Time.now).to_i
-    #   erb :show
-    # end
-
-    # get "/graph" do
-    #   @from    = (params[:from]  || Time.now).to_i
-    #   @time    = params[:time]   || 'minute'
-    #   @targets = params[:target]
-    #   @data_points = prepare_data_points(@from, @time, *@targets)
-    #   @series = @data_points
-    #   erb :graph, :layout => false
-    # end
-
     private
 
-    # params[:id]
-    def to_bson_id(id) 
-      BSON::ObjectId.from_string(id) 
-    end 
-
     def prepare_data_points(from, time, *targets)
-      to = from - time_range(time)
-      result = SimpleMetrics::Graph.query_all(bucket(time), to, from, *targets)
-      result.map do |data_point|
-        { :name => data_point.first, :data => data_point.last.map { |p| { :x => p[:ts], :y => p[:value] || 0 } } }
-      end
-    end
-
-    def one_minute
-      60
-    end
-
-    def one_hour
-      one_minute * 60
-    end
-
-    def one_day
-      one_hour * 24
-    end
-
-    def one_week
-      one_day * 7
-    end
-
-    def time_range(time)
-      case time
-      when 'minute'
-        5*one_minute
-      when 'hour'
-        one_hour
-      when 'day'
-        one_day
-      when 'week'
-        one_week
-      else 
-        raise "Unknown time param: #{time}"
-      end
+      bucket = SimpleMetrics::Bucket.for_time(time)
+      to = from - SimpleMetrics::Graph.time_range(time)
+      SimpleMetrics::Graph.query_all(bucket, to, from, *targets)
     end
 
-    def bucket(time)
-      case time
-      when 'minute'
-        SimpleMetrics::Bucket[0]
-      when 'hour'
-        SimpleMetrics::Bucket[1]
-      when 'day'
-        SimpleMetrics::Bucket[2]
-      when 'week'
-        SimpleMetrics::Bucket[3]
-      end
-    end
   end
 end

data/lib/simple_metrics/bucket.rb

@@ -11,80 +11,33 @@ module SimpleMetrics
       def first
         all.first
       end
-      alias :finest :first
 
       def [](index)
         all[index]
       end
 
-      def flush_raw_data(data)      
-        data_points = []
-        data.each do |str|
-          begin
-            data_points << DataPoint.parse(str)
-          rescue DataPoint::ParserError => e
-            SimpleMetrics.logger.debug "Invalid Data skipped: #{str}, #{e}"
-          end
+      def for_time(time)
+        case time
+        when 'minute'
+          self[0]
+        when 'hour'
+          self[1]
+        when 'day'
+          self[2]
+        when 'week'
+          self[3]
         end
-        flush_data_points(data_points, Time.now.utc.to_i)
       end
 
-      def flush_data_points(data_points, ts = nil)
-        return if data_points.empty?
-        SimpleMetrics.logger.info "#{Time.now} Flushing #{data_points.count} counters to MongoDB"
-
-        ts   ||= Time.now.utc.to_i
-        bucket = Bucket.first
-
-        data_points.group_by { |dp| dp.name }.each_pair do |name,dps|
-          dp = ValueAggregation.aggregate(dps)
-          bucket.save(dp, ts)
-          update_metric(dp)
-          aggregate(dp)  
-        end
-      end
-
-      def aggregate(dp)
-        coarse_buckets.each do |bucket|
-          existing_dp = bucket.find_data_point_at_ts(dp.ts, dp.name)
-          if existing_dp
-            UpdateAggregation.aggregate(existing_dp, dp)
-            bucket.update(existing_dp, existing_dp.ts)
-          else
-            dp.sum   = dp.value
-            dp.total = 1
-            bucket.save(dp, dp.ts)
-          end
-        end
-      end
-
-      private
-
-      def update_metric(dp)
-        metric = MetricRepository.find_one_by_name(dp.name)
-        if metric
-          MetricRepository.update(Metric.new(:name => dp.name, :total => metric.total + 1))
-        else
-          MetricRepository.save(Metric.new(:name => dp.name, :total => 1))
-        end
-      end
-
-      def coarse_buckets
-        Bucket.all.sort_by! { |r| r.seconds }[1..-1]
-      end
-
-      def humanized_timestamp(ts)
-        Time.at(ts).utc
-      end
     end
 
     attr_reader :name, :capped
 
     def initialize(attributes)
-      @name    = attributes['name']
-      @seconds = attributes['seconds']
-      @capped  = attributes['capped']
-      @size    = attributes['size']
+      @name    = attributes.fetch(:name)
+      @seconds = attributes.fetch(:seconds)
+      @capped  = attributes.fetch(:capped)
+      @size    = attributes.fetch(:size)
     end
 
     def seconds
@@ -116,11 +69,6 @@ module SimpleMetrics
       repository.find_data_point_at_ts(ts_bucket(ts), name)
     end
 
-    # TODO: only used in tests, do we need it?
-    def find_all_in_ts_range(from, to)
-      repository.find_all_in_ts_range(from, to)
-    end
-
     def find_all_in_ts_range_by_name(from, to, name)
       repository.find_all_in_ts_range_by_name(from, to, name)
     end
@@ -129,54 +77,32 @@ module SimpleMetrics
       repository.find_all_in_ts_range_by_wildcard(from, to, target)
     end
 
-    # TODO: only used in tests, do we need it?
-    def data_points_exist_at_ts?(ts, name)
-      repository.count_for_name_at(ts, name) > 0
-    end
-
-    # TODO: only used in tests, do we need it?
-    def stats_exist_in_previous_ts?(ts)
-      repository.count_at(ts) > 0
-    end
-
-    def find_all_distinct_names
-      repository.find_all_distinct_names
-    end
-
     def save(dp, ts)
-      dp.ts = ts_bucket(ts)
+      dp.ts    = ts_bucket(ts)
+      dp.sum   = dp.value
+      dp.total = 1
       repository.save(dp)
-      SimpleMetrics.logger.debug "SERVER: MongoDB - insert in #{name}: #{dp.inspect}"
     end
 
     def update(dp, ts)
       dp.ts = ts_bucket(ts)
       repository.update(dp, ts)
-      SimpleMetrics.logger.debug "SERVER: MongoDB - update in #{name}: #{dp.inspect}"
     end
 
     def capped?
       @capped == true
     end
 
-    # TODO refactor, move to graph.rb
     def fill_gaps(from, to, query_result)
       return query_result if query_result.nil? || query_result.size == 0
       
-      tmp_hash = DataPoint.ts_hash(query_result)
+      existing_ts_entries =  query_result.inject({}) { |result, dp| result[dp.ts] = dp; result }
       dp_template = query_result.first
 
       result = []
-      each_ts(from, to) do |current_bucket_ts|
-        result <<  
-          if tmp_hash.key?(current_bucket_ts)
-            tmp_hash[current_bucket_ts]
-          else
-            dp       = dp_template.dup
-            dp.value = nil
-            dp.ts    = current_bucket_ts
-            dp
-          end
+      each_ts(from, to) do |ts_bucket|
+        dp = existing_ts_entries[ts_bucket] || DataPoint::Base.new(:name => dp_template.name, :ts => ts_bucket)
+        result << dp
       end
       result
     end
@@ -188,10 +114,10 @@ module SimpleMetrics
     end
 
     def each_ts(from, to)
-      current_bucket_ts = ts_bucket(from)
-      while (current_bucket_ts <= ts_bucket(to))
-        yield(current_bucket_ts)
-        current_bucket_ts = current_bucket_ts + seconds
+      ts_bucket = ts_bucket(from)
+      while (ts_bucket <= ts_bucket(to))
+        yield(ts_bucket)
+        ts_bucket = ts_bucket + seconds
       end
     end
 

data/lib/simple_metrics/configuration.rb

@@ -6,7 +6,7 @@ module SimpleMetrics
     attr_reader :config
 
     def initialize(hash = {}, &block)
-      @config = load_defaults.merge(hash)  
+      @config = load_defaults.merge(symbolize_keys(hash))  
     end
 
     def configure(hash = {}, &block)
@@ -15,7 +15,7 @@ module SimpleMetrics
     end
 
     def db
-      @db ||= config['db']
+      @db ||= config.fetch(:db)
     end
 
     def db=(db)
@@ -23,7 +23,10 @@ module SimpleMetrics
     end
 
     def buckets
-      @buckets ||= config['buckets']
+      @buckets ||= begin
+        tmp = config.fetch(:buckets)
+        tmp.map { |b| symbolize_keys(b)}
+      end
     end
 
     def buckets=(buckets)
@@ -31,7 +34,7 @@ module SimpleMetrics
     end
 
     def server
-      @server ||= config['server']
+      @server ||= config.fetch(:server)
     end
 
     def server=(server)
@@ -39,7 +42,7 @@ module SimpleMetrics
     end
 
     def web
-      @web ||= config['web']
+      @web ||= config.fetch(:web)
     end
 
     def web=(web)
@@ -49,7 +52,7 @@ module SimpleMetrics
     private
 
     def load_defaults
-      @config = load_config
+      @config = symbolize_keys(load_config)
     rescue Errno::ENOENT # not found error
       logger.info "Creating initial config file: #{config_file}"
       FileUtils.cp(default_config_file, config_file)
@@ -72,6 +75,21 @@ module SimpleMetrics
       logger.error "Error reading config file: #{e}"
     end
 
+    def symbolize_keys(hash)
+      hash.inject({}){|result, (key, value)|
+        new_key = case key
+                  when String then key.to_sym
+                  else key
+                  end
+        new_value = case value
+                    when Hash then symbolize_keys(value)
+                    else value
+                    end
+        result[new_key] = new_value
+        result
+      }
+    end
+
     def logger
       SimpleMetrics.logger
     end

data/lib/simple_metrics/data_point/counter.rb

@@ -8,6 +8,13 @@ module SimpleMetrics
         @value = (@value.to_i || 1) * (1.0 / (@sample_rate || 1).to_f)
       end
 
+      def combine(dp)
+        @total += 1
+        @value += dp.value
+        @sum   += dp.value  
+        self
+      end
+
     end
   end
 end

data/lib/simple_metrics/data_point/event.rb

@@ -7,6 +7,10 @@ module SimpleMetrics
         @type = 'ev'
       end
 
+      def combine(dp)
+        raise "Implement me!"
+      end
+      
     end
   end
 end

data/lib/simple_metrics/data_point/gauge.rb

@@ -8,6 +8,12 @@ module SimpleMetrics
         @value = (@value.to_i || 1) * (1.0 / (@sample_rate || 1).to_f)
       end
 
+      def combine(dp)
+        @total += 1
+        @sum   += dp.value
+        @value  = @sum / @total 
+        self
+      end
     end
   end
 end

data/lib/simple_metrics/data_point/timing.rb

@@ -7,6 +7,9 @@ module SimpleMetrics
         @type = 'ms'
       end
 
+      def combine(dp)
+        raise "Implement me!"
+      end
     end
   end
 end

data/lib/simple_metrics/data_point.rb

@@ -26,7 +26,7 @@ module SimpleMetrics
     def build(attributes)
       case attributes[:type]
       when 'c'
-        DataPoint.create_counter(attributes)
+        Counter.new(attributes)
       when 'g'
         Gauge.new(attributes) 
       when 'ms'
@@ -37,22 +37,6 @@ module SimpleMetrics
         raise UnknownTypeError, "Unknown Type Error: #{attributes[:type]}"
       end
     end
-
-    def create_counter(attributes)
-      Counter.new(attributes)
-    end
-
-    def create_gauge(attributes)
-      Gauge.new(attributes) 
-    end
-
-    def create_timing(attributes)
-      Timing.new(attributes)
-    end
-
-    def ts_hash(query_result)
-      query_result.inject({}) { |result, dp| result[dp.ts] = dp; result }
-    end
     
   end
 end

data/lib/simple_metrics/data_point_repository.rb

@@ -19,25 +19,25 @@ module SimpleMetrics
       def ensure_collections_exist
         SimpleMetrics.logger.debug "SERVER: MongoDB - found following collections: #{db.collection_names.inspect}"
         buckets.each do |retention|
-          unless db.collection_names.include?(retention['name'])
-            db.create_collection(retention['name'], :capped => retention['capped'], :size => retention['size']) 
-            SimpleMetrics.logger.debug "SERVER: MongoDB - created collection #{retention['name']}, capped: #{retention['capped']}, size: #{retention['size']}"
+          unless db.collection_names.include?(retention.fetch(:name))
+            db.create_collection(retention.fetch(:name), :capped => retention.fetch(:capped), :size => retention.fetch(:size)) 
+            SimpleMetrics.logger.debug "SERVER: MongoDB - created collection #{retention.fetch(:name)}, capped: #{retention.fetch(:capped)}, size: #{retention.fetch(:size)}"
           end
           
-          db.collection(retention['name']).ensure_index([['ts', ::Mongo::ASCENDING]])
-          SimpleMetrics.logger.debug "SERVER: MongoDB - ensure index on column ts for collection #{retention['name']}"
+          db.collection(retention.fetch(:name)).ensure_index([['ts', ::Mongo::ASCENDING]])
+          SimpleMetrics.logger.debug "SERVER: MongoDB - ensure index on column ts for collection #{retention.fetch(:name)}"
         end 
       end
 
       def truncate_collections
         buckets.each do |retention|
-          if db.collection_names.include?(retention['name'])
-            if retention['capped']
-              collection(retention['name']).drop # capped collections can't remove elements, drop it instead
+          if db.collection_names.include?(retention.fetch(:name))
+            if retention.fetch(:capped)
+              collection(retention.fetch(:name)).drop # capped collections can't remove elements, drop it instead
             else
-              collection(retention['name']).remove
+              collection(retention.fetch(:name)).remove
             end
-            SimpleMetrics.logger.debug "SERVER: MongoDB - truncated collection #{retention['name']}"
+            SimpleMetrics.logger.debug "SERVER: MongoDB - truncated collection #{retention.fetch(:name)}"
           end
         end
       end
@@ -49,7 +49,7 @@ module SimpleMetrics
       end
 
       def retention_names
-        buckets.map { |r| r['name'] }
+        buckets.map { |r| r.fetch(:name) }
       end
 
       def buckets
@@ -79,11 +79,6 @@ module SimpleMetrics
       data_point(result) if result
     end
 
-    def find_all_in_ts_range(from, to)
-      results = @collection.find(range_query(from, to)).to_a
-      data_points(results)
-    end
-
     def find_all_in_ts_range_by_name(from, to, name)
       results = @collection.find({ :name => name }.merge(range_query(from, to))).to_a
       data_points(results)
@@ -94,18 +89,6 @@ module SimpleMetrics
       data_points(results)
     end
 
-    def count_at(ts)
-      @collection.find({ :ts => ts }).count
-    end
-
-    def count_for_name_at(ts, name)
-      @collection.find({ :ts => ts, :name => name }).count
-    end
-
-    def find_all_distinct_names
-      @collection.distinct(:name).to_a
-    end
-
     private
 
     def regexp(target)

data/lib/simple_metrics/graph.rb

@@ -3,28 +3,32 @@ module SimpleMetrics
   module Graph
     extend self
 
-    def minutes
-      Bucket[0]
-    end
-
-    def hours
-      Bucket[1]
-    end
-
-    def day
-      Bucket[2]
-    end
+    def minutes; Bucket[0]; end
+    def hours;   Bucket[1]; end
+    def day;     Bucket[2]; end
+    def week;    Bucket[3]; end
 
-    def week
-      Bucket[3]
+    def time_range(time)
+      case time
+      when 'minute'
+        5 * one_minute
+      when 'hour'
+        one_hour
+      when 'day'
+        one_day
+      when 'week'
+        one_week
+      else 
+        raise "Unknown time param: #{time}"
+      end
     end
-
+    
     def query_all(bucket, from, to, *targets)
-      result = {}
+      results = []
       Array(targets).each do |target|
-        result[target.inspect] = values_only(query(bucket, from, to, target))
+        results << { :name => target, :data => query(bucket, from, to, target).map { |data| { :x => data.ts, :y => data.value || 0 } } }
       end
-      result
+      results
     end
 
     def query(bucket, from, to, target)
@@ -42,12 +46,24 @@ module SimpleMetrics
 
     private 
 
-    def wild_card_query?(target)
-      target.is_a?(String) && target.include?('*')
+    def one_minute
+      60
     end
 
-    def values_only(data_point_array)
-      data_point_array.map { |data| { :ts => data.ts, :value => data.value } }
+    def one_hour
+      one_minute * 60
+    end
+
+    def one_day
+      one_hour * 24
+    end
+
+    def one_week
+      one_day * 7
+    end
+
+    def wild_card_query?(target)
+      target.is_a?(String) && target.include?('*')
     end
   
   end