simple_metrics 0.4.0 → 0.4.2

data/lib/simple_metrics/importer.rb

@@ -0,0 +1,64 @@
+module SimpleMetrics
+  module Importer
+    extend self
+    
+    def flush_raw(data)      
+      data_points = []
+      data.each do |str|
+        begin
+          data_points << DataPoint.parse(str)
+        rescue DataPoint::ParserError => e
+          SimpleMetrics.logger.debug "Invalid Data skipped: #{str}, #{e}"
+        end
+      end
+      flush_data_points(data_points, Time.now.utc.to_i)
+    end
+
+    def flush_data_points(data_points, ts = nil)
+      return if data_points.empty?
+      SimpleMetrics.logger.info "#{Time.now} Flushing #{data_points.count} to MongoDB"
+
+      ts   ||= Time.now.utc.to_i
+      bucket = Bucket.first
+
+      group_by_name(data_points) do |name, dps|
+        dp = DataPoint.aggregate_values(dps)
+        bucket.save(dp, ts)
+        update_metric(dp, dps.size)
+        aggregate_coarse_buckets(dp)  
+      end
+    end
+
+    def aggregate_coarse_buckets(dp)
+      coarse_buckets.each do |bucket|
+        if existing_dp = bucket.find_data_point_at_ts(dp.ts, dp.name)
+          bucket.update(existing_dp.combine(dp), existing_dp.ts)
+        else
+          bucket.save(dp, dp.ts)
+        end
+      end
+    end
+
+    private
+
+    def group_by_name(dps, &block)
+      dps.group_by { |dp| dp.name }.each_pair do |name,dps|
+        block.call(name, dps)
+      end
+    end
+
+    def update_metric(dp, total)
+      if metric = MetricRepository.find_one_by_name(dp.name)
+        metric.total += total
+        MetricRepository.update(metric)
+      else
+        MetricRepository.save(Metric.new(:name => dp.name, :total => total))
+      end
+    end
+
+    def coarse_buckets
+      Bucket.all.sort_by! { |r| r.seconds }[1..-1]
+    end
+    
+  end
+end

data/lib/simple_metrics/instrument.rb

@@ -0,0 +1,28 @@
+module SimpleMetrics
+  class Instrument
+
+    attr_accessor :id, :name, :metrics, :created_at, :updated_at
+
+    def initialize(attributes)
+      @id          = attributes[:id]
+      @name        = attributes[:name]
+      @created_at  = attributes[:created_at]
+      @updated_at  = attributes[:updated_at]
+      @metrics     = attributes[:metrics] || []
+    end
+
+    def attributes
+      { 
+        :id          => @id.to_s, # convert bson id to str
+        :name        => @name,
+        :metrics     => @metrics,
+        :created_at  => @created_at,
+        :updated_at  => @updated_at
+      }
+    end
+
+    def to_s
+      attributes.to_s
+    end
+  end
+end

data/lib/simple_metrics/instrument_repository.rb

@@ -0,0 +1,64 @@
+module SimpleMetrics
+  class InstrumentRepository
+
+    class << self
+
+      def find_one(id)
+        instrument(collection.find_one(BSON::ObjectId.from_string(id)))
+      end
+
+      def find_one_by_name(name)      
+        result = collection.find({ :name => name }).to_a.first
+        instrument(result) if result
+      end
+
+      def find_all
+        results = collection.find.sort([['name', ::Mongo::ASCENDING]]).to_a
+        instruments(results) if results
+      end
+
+      def save(instrument)
+        instrument.created_at = Time.now.utc
+        instrument.updated_at = Time.now.utc
+        attributes = instrument.attributes.reject { |key, value| key.to_s == "id" }
+        id = collection.insert(attributes)
+        instrument.id = id
+        id
+      end
+
+      def update(instrument)
+        collection.update({ "_id" => instrument.id }, "$set" => instrument.attributes.merge(:updated_at => Time.now.utc).reject { |k, v| k == 'id' })
+      end
+
+      def remove(id)
+        collection.remove("_id" => BSON::ObjectId.from_string(id))
+      end
+
+      def truncate_collections
+        collection.remove
+      end
+
+      def ensure_index
+        collection.ensure_index([['created_at', ::Mongo::ASCENDING]])
+        collection.ensure_index([['updated_at', ::Mongo::ASCENDING]])
+        collection.ensure_index([['name', ::Mongo::ASCENDING]])
+      end
+
+      private
+
+      def collection
+        Repository.db.collection('instruments')
+      end
+
+      def instrument(result)
+        Instrument.new(:id => result["_id"], :name => result["name"], :metrics => result["metrics"], :created_at => result["created_at"], :updated_at => result["updated_at"])
+      end
+
+      def instruments(results)
+        results.inject([]) { |result, a| result << instrument(a); }
+      end
+
+    end # class << self
+
+  end
+end

data/lib/simple_metrics/metric.rb

@@ -0,0 +1,29 @@
+module SimpleMetrics
+  class Metric
+
+    attr_reader :id, :name, :created_at, :updated_at
+    attr_accessor :total
+
+    def initialize(attributes)
+      @id          = attributes[:id]
+      @name        = attributes[:name]
+      @total       = attributes[:total]
+      @created_at  = attributes[:created_at]
+      @updated_at  = attributes[:updated_at]
+    end
+
+    def attributes
+      { 
+        :id          => @id.to_s, # convert bson id to str
+        :name        => @name,
+        :total       => @total,
+        :created_at  => @created_at,
+        :updated_at  => @updated_at
+      }
+    end
+
+    def to_s
+      attributes.to_s
+    end
+  end
+end

data/lib/simple_metrics/metric_repository.rb

@@ -0,0 +1,54 @@
+module SimpleMetrics
+  class MetricRepository
+
+    class << self
+
+      def find_one(id)
+        metric(collection.find_one(id))
+      end
+
+      def find_one_by_name(name)      
+        result = collection.find({ :name => name }).to_a.first
+        metric(result) if result
+      end
+
+      def find_all
+        results = collection.find.sort([['name', ::Mongo::ASCENDING]]).to_a
+        metrics(results) if results
+      end
+
+      def save(metric)
+        collection.insert(metric.attributes.merge(:created_at => Time.now.utc, :updated_at => Time.now.utc))
+      end
+
+      def update(metric)
+        collection.update({ "_id" => metric.id }, { "$set" => { :total => metric.total, :updated_at => Time.now.utc }})  
+      end
+
+      def truncate_collections
+        collection.remove
+      end
+
+      def ensure_index
+        collection.ensure_index([['created_at', ::Mongo::ASCENDING]])
+        collection.ensure_index([['updated_at', ::Mongo::ASCENDING]])
+        collection.ensure_index([['name', ::Mongo::ASCENDING]])
+      end
+
+      private
+
+      def collection
+        Repository.db.collection('metrics')
+      end
+
+      def metric(result)
+        Metric.new(:id => result["_id"], :name => result["name"], :total => result["total"], :created_at => result["created_at"], :updated_at => result["updated_at"])
+      end
+
+      def metrics(results)
+        results.inject([]) { |result, a| result << metric(a); }
+      end
+    end
+
+  end
+end

data/lib/simple_metrics/repository.rb

@@ -0,0 +1,34 @@
+module SimpleMetrics
+  module Repository
+    extend self
+
+    def connection
+      @@connection ||= ::Mongo::Connection.new(host, port)
+    end
+
+    def db
+      @@db ||= connection.db(db_name, config.fetch(:options))
+    end
+
+    def db_name
+      "simple_metrics_#{prefix}"
+    end
+
+    def prefix
+      config[:prefix] || 'development'
+    end
+    
+    def host
+      config[:host] || 'localhost'
+    end
+
+    def port
+      config[:port] || 27017
+    end
+
+    def config
+      SimpleMetrics.config.db
+    end
+
+  end
+end

data/lib/simple_metrics/udp_server.rb

@@ -0,0 +1,81 @@
+# encoding: utf-8
+require "eventmachine"
+
+if defined? Encoding
+  Encoding.default_external = Encoding::UTF_8
+end
+
+module SimpleMetrics
+
+  module ClientHandler
+
+    @@data = []
+
+    class << self
+      def get_and_clear_data
+        data = @@data.dup
+        @@data = []
+        data
+      end
+    end
+
+    def post_init
+      SimpleMetrics.logger.info "ClientHandler entering post_init"
+    end
+
+    def receive_data(data)
+      SimpleMetrics.logger.debug "received_data: #{data.inspect}"
+      @@data << data
+    end
+  end
+
+  class UDPServer
+
+    def start
+      SimpleMetrics.logger.info "SERVER: starting up on #{host}:#{port}..."
+
+      trap('TERM') { stop }
+      trap('INT')  { stop }
+
+      DataPointRepository.ensure_collections_exist
+
+      EM.run do
+        EM.open_datagram_socket(host, port, SimpleMetrics::ClientHandler) do |con|
+          EventMachine::add_periodic_timer(flush_interval) do
+            SimpleMetrics.logger.debug "SERVER: period timer triggered after #{flush_interval} seconds"
+
+            EM.defer { Bucket.flush_raw(ClientHandler.get_and_clear_data) } 
+          end
+        end
+      end
+    end
+
+    def stop
+      SimpleMetrics.logger.info "EventMachine stop"
+      EM.stop
+    end
+    
+    def to_s
+      "#{host}:#{port}"
+    end
+
+    private
+
+    def host
+      config['host'] || 'localhost'
+    end
+
+    def port 
+      config['port'] || 8125
+    end
+
+    def flush_interval
+      config['flush_interval'] || 10
+    end
+
+    def config
+      SimpleMetrics.config.server
+    end
+
+  end
+end

data/lib/simple_metrics/version.rb

@@ -1,4 +1,4 @@
 # encoding: utf-8
 module SimpleMetrics
-  VERSION = "0.4.0"
+  VERSION = "0.4.2"
 end

data/lib/simple_metrics.rb

@@ -1,14 +1,28 @@
 # encoding: utf-8
 require "logger"
+require "active_support/all"
 
 require "simple_metrics/version"
-require "simple_metrics/client"
-require "simple_metrics/server"
+require "simple_metrics/configuration"
+require "simple_metrics/udp_server"
+require "simple_metrics/repository"
+require "simple_metrics/data_point_repository"
 require "simple_metrics/data_point"
+require "simple_metrics/data_point/base"
+require "simple_metrics/data_point/counter"
+require "simple_metrics/data_point/event"
+require "simple_metrics/data_point/gauge"
+require "simple_metrics/data_point/timing"
+require "simple_metrics/importer"
 require "simple_metrics/bucket"
 require "simple_metrics/graph"
 require "simple_metrics/functions"
-require "simple_metrics/mongo"
+require "simple_metrics/metric"
+require "simple_metrics/metric_repository"
+require "simple_metrics/instrument"
+require "simple_metrics/instrument_repository"
+require "simple_metrics/dashboard"
+require "simple_metrics/dashboard_repository"
 
 module SimpleMetrics
   extend self
@@ -21,81 +35,12 @@ module SimpleMetrics
     @@logger = logger
   end
 
-  CONFIG_DEFAULTS = {
-    :host           => 'localhost',
-    :port           => 8125,
-    :flush_interval => 10
-  }.freeze
-
   def config
-    @@config ||= CONFIG_DEFAULTS
-  end
-
-  def config=(options)
-    @@config = CONFIG_DEFAULTS.merge(options)
-  end
-
-  BUCKETS_DEFAULTS = [
-    { 
-      :name    => 'stats_per_10s',
-      :seconds => 10,
-      :capped  => true,
-      :size    => 100_100_100
-    },
-    {
-      :name    => 'stats_per_1min',
-      :seconds => 60,
-      :capped  => true,
-      :size    => 1_100_100_100
-    },
-    {
-      :name    => 'stats_per_10min',
-      :seconds => 600,
-      :size    => 0 ,
-      :capped  => false
-    },
-    {
-      :name    => 'stats_per_day',
-      :seconds => 86400,
-      :size    => 0,
-      :capped  => false
-    }
-  ].freeze
-
-  def buckets_config
-    @@buckets ||= BUCKETS_DEFAULTS
-  end
-
-  def buckets_config=(buckets)
-    @@buckets = buckets
-  end
-
-  MONGODB_DEFAULTS = {
-    :pool_size => 5, 
-    :timeout   => 5,
-    :strict    => true
-  }.freeze
-
-  DB_CONFIG_DEFAULTS = {
-    :host      => 'localhost',
-    :port      => 27017,
-    :prefix    => 'development'
-  }.freeze
-
-  def db_config=(options)
-    @@db_config = {
-      :host    => options.delete(:host) || 'localhost',
-      :port    => options.delete(:port) || 27017,
-      :db_name => "simple_metrics_#{options.delete(:prefix)}",
-      :options => MONGODB_DEFAULTS.merge(options)
-    }
+    @@config ||= Configuration.new
   end
 
-  def db_config
-    @@db_config ||= DB_CONFIG_DEFAULTS.merge(
-      :db_name => "simple_metrics_#{DB_CONFIG_DEFAULTS[:prefix]}",
-      :options => MONGODB_DEFAULTS
-    )
+  def configure(hash = {}, &block)
+    config.configure(hash, &block)
   end
 
-end
+end

data/simple_metrics.gemspec

@@ -19,9 +19,13 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rake"
   s.add_development_dependency "rspec"
   s.add_development_dependency "rr"
+  s.add_development_dependency "shotgun"
+  s.add_development_dependency "rack-test"
 
   s.add_dependency "eventmachine"
   s.add_dependency "daemons"
   s.add_dependency "mongo", '~> 1.6'
   s.add_dependency "bson", '~> 1.6'
+  s.add_dependency "bson_ext", '~> 1.6'
+  s.add_dependency "activesupport"
 end