simple_metrics 0.3.6 → 0.4.0

data/lib/simple_metrics/client.rb

@@ -0,0 +1,83 @@
+# encoding: utf-8
+require "socket"
+
+module SimpleMetrics
+
+  class Client
+    VERSION = "0.0.1"
+    
+    def initialize(host, port = 8125)
+      @host, @port = host, port
+    end
+
+    # send relative value
+    def increment(stat, sample_rate = 1)
+      count(stat, 1, sample_rate)
+    end
+
+    # send relative value
+    def decrement(stat, sample_rate = 1)
+      count(stat, -1, sample_rate)
+    end
+
+    # send relative value
+    def count(stat, count, sample_rate = 1)
+      send_data( stat, count, 'c', sample_rate)
+    end
+
+    # send absolute value
+    # TODO: check if this is actually supported by Statsd server
+    def gauge(stat, value)
+      send_data(stat, value, 'g')
+    end
+
+    # Sends a timing (in ms) (glork)
+    def timing(stat, ms, sample_rate = 1)
+      send_data(stat, ms, 'ms', sample_rate)
+    end
+
+    # Sends a timing (in ms) block based
+    def time(stat, sample_rate = 1, &block)
+      start = Time.now
+      result = block.call
+      timing(stat, ((Time.now - start) * 1000).round, sample_rate)
+      result
+    end
+
+    private
+
+    def sampled(sample_rate, &block)
+      if sample_rate < 1
+        block.call if rand <= sample_rate
+      else
+        block.call
+      end
+    end
+
+    def send_data(stat, delta, type, sample_rate = 1)
+      sampled(sample_rate) do
+        data = "#{stat}:#{delta}|#{type}" # TODO: check stat is valid
+        data << "|@#{sample_rate}" if sample_rate < 1
+        send_to_socket(data)
+      end
+    end
+
+    def send_to_socket(data)
+      logger.debug "SimpleMetrics Client send: #{data}"
+      socket.send(data, 0, @host, @port)
+    rescue Exception => e
+      puts e.backtrace
+      logger.error "SimpleMetrics Client error: #{e}"
+    end
+
+    def socket
+      @socket ||= UDPSocket.new 
+    end
+
+    def logger
+      @logger ||= SimpleMetrics.logger
+    end
+
+  end
+
+end

data/lib/simple_metrics/data_point.rb

@@ -1,10 +1,10 @@
+# encoding: utf-8
 module SimpleMetrics
-  module DataPoint
-    extend self
+
+  class DataPoint
 
     class NonMatchingTypesError < Exception; end
     class ParserError           < Exception; end
-    class UnknownTypeError      < Exception; end
 
     # examples:
     # com.example.test1:1|c
@@ -14,57 +14,149 @@ module SimpleMetrics
     # com.example.test4:44|ms
     REGEXP = /^([\d\w_.]*):(-?[\d]*)\|(c|g|ms){1}(\|@([.\d]+))?$/i
 
-    def parse(str)
-      if str =~ REGEXP
-        name, value, type, sample_rate = $1, $2, $3, $5
-        build(:name => name, :value => value, :type => type, :sample_rate => sample_rate)
-      else
-        raise ParserError, "Parser Error - Invalid data point: #{str}"
+    class << self
+
+      def parse(str)
+        if str =~ REGEXP
+          name, value, type, sample_rate = $1, $2, $3, $5
+          if type == "ms"
+            # TODO: implement sample_rate handling
+            create_timing(:name => name, :value => value)
+          elsif type == "g"
+            create_gauge(:name => name, :value => (value.to_i || 1) * (1.0 / (sample_rate || 1).to_f) )
+          elsif type == "c"
+            create_counter(:name => name, :value => (value.to_i || 1) * (1.0 / (sample_rate || 1).to_f) )
+          end
+        else
+          raise ParserError, "Parser Error - Invalid Stat: #{str}"
+        end
       end
-    end
 
-    def build(attributes)
-      case attributes[:type]
-      when 'c'
-        Counter.new(attributes)
-      when 'g'
-        Gauge.new(attributes) 
-      when 'ms'
-        Timing.new(attributes)
-      when 'ev'
-        Event.new(attributes)
-      else
-        raise UnknownTypeError, "Unknown Type Error: #{attributes[:type]}"
+      def create_counter(attributes)
+        self.new(attributes.merge(:type => 'c'))
       end
-    end
-    
-    def aggregate_values(dps)
-      raise SimpleMetrics::DataPoint::NonMatchingTypesError if has_non_matching_types?(dps)
-
-      dp       = dps.first.dup
-      dp.value = if dp.counter?
-        sum(dps)
-      elsif dp.gauge?
-        sum(dps) / dps.size
-      elsif dp.event?
-        raise "Implement me!"
-      elsif dp.timing?
-        raise "Implement me!"
-      else
-        raise ArgumentError("Unknown data point type: #{dp}")
+
+      def create_gauge(attributes)
+        self.new(attributes.merge(:type => 'g'))
+      end
+
+      def create_timing(attributes)
+        self.new(attributes.merge(:type => 'ms'))
+      end
+
+      def aggregate(stats_array, name = nil)
+        raise NonMatchingTypesError unless stats_array.group_by { |stats| stats.type }.size == 1
+
+        result_stat      = stats_array.first.dup
+        result_stat.name = name if name
+        if stats_array.first.counter?
+          result_stat.value = stats_array.map { |stats| stats.value }.inject(0) { |result, value| result += value }
+          result_stat
+        elsif stats_array.first.gauge?
+          total_value = stats_array.map { |stats| stats.value }.inject(0) { |result, value| result += value }
+          result_stat.value = total_value / stats_array.size
+          result_stat
+        elsif stats_array.first.timing?
+          # TODO implement timing aggregation
+        elsif stats_array.first.event? 
+          # TODO implement event aggregation
+        else
+          raise ArgumentError, "Unknown data point type"
+        end
+      end
+
+      def aggregate_array(stats_array, name = nil)
+        raise NonMatchingTypesError unless stats_array.group_by { |stats| stats.type }.size == 1
+
+        if stats_array.first.counter?
+          tmp_hash = ts_hash_aggregated(stats_array) do |value1, value2|
+            value1 + value2
+          end
+
+          result = []
+          tmp_hash.each_pair do |key, value|
+            result << self.new(:name => name, :ts => key, :value => value, :type => 'c')
+          end
+          result
+        elsif stats_array.first.gauge?
+          tmp_hash = ts_hash_aggregated(stats_array) do |value1, value2|
+            (value1 + value2)/2
+          end
+
+          result = []
+          tmp_hash.each_pair do |key, value|
+            result << self.new(:name => name, :ts => key, :value => value, :type => 'g')
+          end
+          result
+        elsif stats_array.first.timing?
+          # TODO implement timing aggregation
+        elsif stats_array.first.event? 
+          # TODO implement event aggregation
+        else
+          raise ArgumentError, "Unknown data point type"
+        end
       end
-      dp
+
+      def create_from_db(attributes)
+        self.new(:name => attributes["name"], :value => attributes["value"], :ts => attributes["ts"], :type => attributes["type"])
+      end
+
+      def ts_hash(query_result)
+        query_result.inject({}) { |result, dp| result[dp.ts] = dp; result }
+      end
+      
+      private
+
+      def ts_hash_aggregated(data_points, &block)
+        tmp = {}
+        data_points.each do |dp|
+          if tmp.key?(dp.ts)
+            tmp[dp.ts] = block.call(tmp[dp.ts], dp.value)
+          else
+            tmp[dp.ts] = dp.value
+          end
+        end
+        tmp
+      end
+    end
+
+    attr_accessor :name, :ts, :type, :value
+
+    def initialize(attributes)
+      @name  = attributes[:name]
+      @value = attributes[:value]
+      @ts    = attributes[:ts]
+      @type  = attributes[:type]
     end
 
-    private
+    def counter?
+      type == 'c'
+    end
+
+    def gauge?
+      type == 'g'
+    end
+
+    def timing?
+      type == 'ms'
+    end
+
+    def timestamp
+      ts
+    end
 
-    def sum(dps)
-      dps.map { |dp| dp.value }.inject(0) { |result, value| result += value }
+    def value
+      @value.to_i if @value
     end
 
-    def has_non_matching_types?(dps)
-      dps.group_by { |dp| dp.type }.size != 1
+    def attributes
+      { 
+        :name  => name,
+        :value => value,
+        :ts    => ts,
+        :type  => type
+      }
     end
 
   end
-end
+end

data/lib/simple_metrics/functions.rb

@@ -2,14 +2,14 @@ module SimpleMetrics
 
 	module Functions
 
-  	# calculate the maximum value for multiple targets
-  	#
-  	# params:
+		# calculate the maximum value for multiple targets
+		#
+		# params:
     #  data_points   [1, 3, 5], [2, 1, 6]
     #
     # return:
     #  array         [2, 3, 6]
-	 def max(*data_points)
+		def max(*data_points)
     end
 
     # calculate the minimum value for multiple targets
@@ -17,7 +17,7 @@ module SimpleMetrics
     end
 
     # add offset to each value
-	 def offset(*data_points)
+		def offset(*data_points)
     end
 
     # multiple each value 

data/lib/simple_metrics/graph.rb

@@ -1,95 +1,58 @@
 module SimpleMetrics
 
+  # 
+  # url format examples:
+  # * target=com.post.clicks (1 line in graph)
+  # * target=com.post.clicks.text&target=com.post.clicks.logo (2 lines in graph)
+  # * target=com.post.clicks.* (1 aggregated line in graph)
+  #
   module Graph
     extend self
 
-    def minutes; Bucket[0]; end
-    def hours;   Bucket[1]; end
-    def day;     Bucket[2]; end
-    def week;    Bucket[3]; end
+    def minutes
+      Bucket[0]
+    end
 
-    def time_range(time)
-      case time
-      when 'minute'
-        5 * one_minute
-      when 'hour'
-        one_hour
-      when 'day'
-        one_day
-      when 'week'
-        one_week
-      else 
-        raise "Unknown time param: #{time}"
-      end
+    def hours
+      Bucket[1]
+    end
+
+    def day
+      Bucket[2]
     end
-    
+
+    def week
+      Bucket[3]
+    end
+
     def query_all(bucket, from, to, *targets)
-      results = []
+      result = {}
       Array(targets).each do |target|
-        results << { :name => target, :data => query(bucket, from, to, target).map { |data| { :x => data.ts, :y => data.value || 0 } } }
+        result[target.inspect] = values_only(query(bucket, from, to, target))
       end
-      results
+      result
     end
 
     def query(bucket, from, to, target)
-      if wild_card_query?(target)
+      if target.is_a?(Regexp) 
+        result = bucket.find_all_in_ts_range_by_regexp(from, to, target)
+        result = DataPoint.aggregate_array(result, target.inspect)
+        bucket.fill_gaps(from, to, result)
+      elsif target.is_a?(String) && target.include?('*')
         result = bucket.find_all_in_ts_range_by_wildcard(from, to, target)
-        result = aggregate(result, target)
+        result = DataPoint.aggregate_array(result, target)
         bucket.fill_gaps(from, to, result)
       elsif target.is_a?(String)
         result = bucket.find_all_in_ts_range_by_name(from, to, target)
         bucket.fill_gaps(from, to, result) 
       else
-        raise ArgumentError, "Unknown target format: #{target.inspect}"
-      end
-    end
-
-    private 
-
-    def aggregate(dps, target)
-      raise SimpleMetrics::DataPoint::NonMatchingTypesError if has_non_matching_types?(dps)
-
-      tmp = {}
-      dps.each do |dp|
-        dp.name = target
-        if tmp.key?(dp.ts)
-          tmp[dp.ts] << dp
-        else
-          tmp[dp.ts] = [dp]
-        end
-      end
-      tmp
-
-      result = []
-      tmp.each_pair do |ts, dps|
-        result << DataPoint.aggregate_values(dps)
+        raise ArgumentError, "Unknown target: #{target.inspect}"
       end
-      result
     end
 
-    def has_non_matching_types?(dps)
-      dps.group_by { |dp| dp.type }.size != 1
+    def values_only(data_point_array)
+      data_point_array.map { |data| { :ts => data.ts, :value => data.value } }
     end
 
-    def one_minute
-      60
-    end
-
-    def one_hour
-      one_minute * 60
-    end
-
-    def one_day
-      one_hour * 24
-    end
-
-    def one_week
-      one_day * 7
-    end
-
-    def wild_card_query?(target)
-      target.is_a?(String) && target.include?('*')
-    end
-  
   end
 end

data/lib/simple_metrics/mongo.rb

@@ -0,0 +1,48 @@
+# encoding: utf-8
+require "mongo"
+
+module SimpleMetrics
+  module Mongo
+    extend self
+
+    def ensure_collections_exist
+      SimpleMetrics.logger.debug "SERVER: MongoDB - found following collections: #{db.collection_names.inspect}"
+      Bucket.all.each do |bucket|
+        unless db.collection_names.include?(bucket.name)
+          db.create_collection(bucket.name, :capped => bucket.capped, :size => bucket.size) 
+          SimpleMetrics.logger.debug "SERVER: MongoDB - created collection #{bucket.name}, capped: #{bucket.capped}, size: #{bucket.size}"
+        end
+        
+        db.collection(bucket.name).ensure_index([['ts', ::Mongo::ASCENDING]])
+        SimpleMetrics.logger.debug "SERVER: MongoDB - ensure index on column ts for collection #{bucket.name}"
+      end 
+    end
+
+    def truncate_collections
+      Bucket.all.each do |bucket|
+        if db.collection_names.include?(bucket.name)
+          if bucket.capped?
+            collection(bucket.name).drop # capped collections can't remove elements, drop it instead
+          else
+            collection(bucket.name).remove
+          end
+          SimpleMetrics.logger.debug "SERVER: MongoDB - truncated collection #{bucket.name}"
+        end
+      end
+    end
+
+    @@collection = {}
+    def collection(name)
+      @@collection[name] ||= db.collection(name)
+    end
+
+    def connection
+      @@connection ||= ::Mongo::Connection.new(SimpleMetrics.db_config[:host], SimpleMetrics.db_config[:port])
+    end
+
+    def db
+      @@db ||= connection.db(SimpleMetrics.db_config[:db_name], SimpleMetrics.db_config[:options])
+    end
+
+  end
+end