simple_metrics 0.3.6 → 0.4.0

data/README.markdown

@@ -11,39 +11,76 @@ SimpleMetrics is written in Ruby and packaged as a gem.
 
 The current version is considered ALPHA.
 
-SimpleMetrics Server
+SimpleMetrics Client
 --------------------
 
-We provide a simple commandline wrapper using daemons gem (http://daemons.rubyforge.org/).
+Commandline client:
 
-Start Server as background daemon:
+Send a count of 5 for data point "module.test1":
 
-    simple_metrics_server start
+		simple_metrics_client module.test1 -counter 5
 
-Start in foreground:
+Send a timing of 100ms:
 
-    simple_metrics_server start -t
+		simple_metrics_client module.test1 -timing 100
 
-Show Help:
+doing the same, but since we expect a lot of calls we sample the data (10%):
+
+		simple_metrics_client module.test1 -timing 100 --sample_rate 0.1
+
+more info:
+	
+		simple_metrics_client --help
+
+Ruby client API
+---------------
+
+Initialize client:
+
+		client = SimpleMetrics::Client.new("localhost")
+
+sends "com.example.test1:1|c" via UDP:
+
+		client.increment("com.example.test1")
+
+sends "com.example.test1:-1|c":
 
-    simple_metrics_server --help
+		client.decrement("com.example.test1")
 
-SimpleMetrics Web App
------------------
+sends "com.example.test1:5|c" (a counter with a relative value of 5):
 
-A small Sinatra app is provided using the vegas gem (https://github.com/quirkey/vegas).
+		client.count("com.example.test1", 5)
+
+sends "com.example.test1:5|c|@0.1" with a sample rate of 10%:
+
+		client.count("com.example.test1", 5, 0.1)
+
+sends "com.example.test1:5|g" (meaning gauge, an absolute value of 5):
+
+		client.count("com.example.test1", 5)
+
+sends "com.example.test1:100|ms":
+
+		client.timing("com.example.test1")
+
+More examples in the examples/ directory.
+
+SimpleMetrics Server
+--------------------
+
+We provide a simple commandline wrapper using daemons gem (http://daemons.rubyforge.org/).
 
-Start web app as background daemon:
+Start Server as background daemond:
 
-    simple_metrics_web
+		simple_metrics_server start
 
 Start in foreground:
 
-    simple_metrics_web -F
+		simple_metrics_server start -t
 
 Show Help:
 
-    simple_metrics_web --help
+		simple_metrics_server --help
 
 Round Robin Database Principles in MongoDB
 ------------------------------------------
@@ -54,7 +91,7 @@ We use 4 collections in MongoDB each with more coarse timestamp buckets:
 * 10 min
 * 1 day
 
-The 10sec and 1min collections are capped collections and have a fixed size. The others will store the data as long as there is sufficient disc space.
+The 10s and 1m collections are capped collections and have a fixed size. The other will store the data as long as we have sufficient disc space.
 
 How can we map these times to graphs?
 

data/Rakefile

@@ -2,24 +2,7 @@ require "bundler/gem_tasks"
 require 'rake'
 require 'rspec/core/rake_task'
 
-require 'simple_metrics'
-
 RSpec::Core::RakeTask.new 
 task :default  => :spec
 task :test  => :spec
 
-namespace :simple_metrics do
-
-  desc "Ensure collections and index exist"
-  task :ensure_collections_exist do
-    SimpleMetrics.logger = Logger.new($stdout)
-    SimpleMetrics::DataPointRepository.ensure_collections_exist
-  end
-
-  desc "Truncate all collections"
-  task :truncate_collections do
-    SimpleMetrics.logger = Logger.new($stdout)
-    SimpleMetrics::DataPointRepository.truncate_collections
-  end
-
-end

data/bin/populate

@@ -6,33 +6,20 @@ require "bundler/setup"
 require 'optparse'
 require "simple_metrics"
 
-SimpleMetrics.logger = Logger.new("/dev/null")
-SimpleMetrics::DataPointRepository.truncate_collections
-SimpleMetrics::DataPointRepository.ensure_collections_exist
+ts = Time.now.to_i
 
-bucket = SimpleMetrics::Bucket.first
+name = "test.post.clicks2"
 
-name = ENV['NAME'] || "test.page.visits.1"
-now = Time.now.to_i
-minute = 60
-hour   = minute * 60
+SimpleMetrics::Mongo.ensure_collections_exist
 
-def create_dps(name)
-  previous_value = 20
-  (1..1).inject([]) do |result, index|
-    value = previous_value+rand(20)
-    result << SimpleMetrics::DataPoint::Counter.new(:name => name, :value => value)
-    previous_value = value
-    result
-  end
-end
+SimpleMetrics::Bucket.all.each do |bucket|
+	ts_bucket = bucket.ts_bucket(ts)
 
-counter = 1
-current = now - 1 * hour
-while (current < now)
-  dps = create_dps(name)
-  puts "flush data for #{Time.at(current)}, #{counter}"
-  SimpleMetrics::Importer.flush_data_points(dps, current)
-  current += 10
-  counter += 1
-end
+	previous_value = rand(300)
+	(1..100).each do |index|
+		ts = ts_bucket - (bucket.seconds*index)
+		value = previous_value+rand(20)
+		data_point = SimpleMetrics::DataPoint.create_counter(:name => name, :value => value)
+		bucket.save(data_point, ts)
+	end
+end

data/bin/simple_metrics_client

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "bundler/setup"
+
+require 'optparse'
+require "simple_metrics"
+
+options = {
+	:host        => 'localhost',
+	:port        => 8125,
+	:sample_rate => 1
+}
+
+parser ||= OptionParser.new do |opts|
+  opts.banner = "Usage Example: simple_metrics_send com.test.mymetric -c5"
+
+  opts.separator ""
+  opts.separator "Client options:"
+
+  opts.on("-c", "--counter VALUE", "Counter, a relative value") do |value|
+  	options[:type] = 'c'
+  	options[:stat] = value.to_i
+  end
+  opts.on("-g", "--gauge VALUE", "Gauge, an absolute value ") do |value|
+  	options[:type] = 'g'
+  	options[:stat] = value.to_i
+  end
+  opts.on("-t", "--timing VALUE", "A timing in ms") do |value|
+  	options[:type] = 'ms'
+  	options[:stat] = value.to_i
+  end
+  opts.on("-s", "--sample_rate VALUE", "An optional sample rate between 0 and 1 (example: 0.2)") do |value|
+  	options[:sample_rate] = value.to_f || 1
+  end
+
+  opts.separator ""
+
+  opts.on("-a", "--address HOST", "bind to HOST address (default: #{options[:host]})") do |host| 
+  	options[:host] = host
+  end
+
+  opts.on("-p", "--port PORT", "use PORT (default: #{options[:port]})") do |port|
+  	options[:port] = port.to_i
+  end
+
+  opts.separator ""
+  opts.on_tail("-h", "--help", "Show this message") { puts opts; exit }
+  opts.on_tail('-v', '--version', "Show version") { puts SimpleMetrics::VERSION; exit }
+
+end.parse!(ARGV)
+
+command    = ARGV.shift
+arguments  = ARGV
+client = SimpleMetrics::Client.new(options[:host])
+
+case options[:type]
+when'c' 
+	client.count(command, options[:stat], options[:sample_rate])
+when 'g'
+	client.gauge(command, options[:stat], options[:sample_rate])
+when 'ms'
+	client.timing(command, options[:stat], options[:sample_rate])
+end

data/bin/simple_metrics_server

@@ -1,19 +1,18 @@
 #!/usr/bin/env ruby
 
-$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
-
 require "rubygems"
+require "bundler/setup"
 require "simple_metrics"
 require "daemons"
 
 options = {
 	:backtrace  => true,
 	:log_output => true,
-	:dir_mode   => :script,
+	:dir_mode   => :script
 }
 
 Daemons.run_proc("simple_metrics", options) do
-	SimpleMetrics::UDPServer.new.start
+	SimpleMetrics::Server.new.start
 end
 
 

data/lib/simple_metrics.rb

@@ -2,23 +2,13 @@
 require "logger"
 
 require "simple_metrics/version"
-require "simple_metrics/configuration"
-require "simple_metrics/udp_server"
-require "simple_metrics/repository"
-require "simple_metrics/data_point_repository"
+require "simple_metrics/client"
+require "simple_metrics/server"
 require "simple_metrics/data_point"
-require "simple_metrics/data_point/base"
-require "simple_metrics/data_point/counter"
-require "simple_metrics/data_point/event"
-require "simple_metrics/data_point/gauge"
-require "simple_metrics/data_point/timing"
-require "simple_metrics/importer"
 require "simple_metrics/bucket"
 require "simple_metrics/graph"
 require "simple_metrics/functions"
-require "simple_metrics/metric"
-require "simple_metrics/metric_repository"
-require "simple_metrics/app"
+require "simple_metrics/mongo"
 
 module SimpleMetrics
   extend self
@@ -31,12 +21,81 @@ module SimpleMetrics
     @@logger = logger
   end
 
+  CONFIG_DEFAULTS = {
+    :host           => 'localhost',
+    :port           => 8125,
+    :flush_interval => 10
+  }.freeze
+
   def config
-    @@config ||= Configuration.new
+    @@config ||= CONFIG_DEFAULTS
+  end
+
+  def config=(options)
+    @@config = CONFIG_DEFAULTS.merge(options)
+  end
+
+  BUCKETS_DEFAULTS = [
+    { 
+      :name    => 'stats_per_10s',
+      :seconds => 10,
+      :capped  => true,
+      :size    => 100_100_100
+    },
+    {
+      :name    => 'stats_per_1min',
+      :seconds => 60,
+      :capped  => true,
+      :size    => 1_100_100_100
+    },
+    {
+      :name    => 'stats_per_10min',
+      :seconds => 600,
+      :size    => 0 ,
+      :capped  => false
+    },
+    {
+      :name    => 'stats_per_day',
+      :seconds => 86400,
+      :size    => 0,
+      :capped  => false
+    }
+  ].freeze
+
+  def buckets_config
+    @@buckets ||= BUCKETS_DEFAULTS
+  end
+
+  def buckets_config=(buckets)
+    @@buckets = buckets
+  end
+
+  MONGODB_DEFAULTS = {
+    :pool_size => 5, 
+    :timeout   => 5,
+    :strict    => true
+  }.freeze
+
+  DB_CONFIG_DEFAULTS = {
+    :host      => 'localhost',
+    :port      => 27017,
+    :prefix    => 'development'
+  }.freeze
+
+  def db_config=(options)
+    @@db_config = {
+      :host    => options.delete(:host) || 'localhost',
+      :port    => options.delete(:port) || 27017,
+      :db_name => "simple_metrics_#{options.delete(:prefix)}",
+      :options => MONGODB_DEFAULTS.merge(options)
+    }
   end
 
-  def configure(hash = {}, &block)
-    config.configure(hash, &block)
+  def db_config
+    @@db_config ||= DB_CONFIG_DEFAULTS.merge(
+      :db_name => "simple_metrics_#{DB_CONFIG_DEFAULTS[:prefix]}",
+      :options => MONGODB_DEFAULTS
+    )
   end
 
-end
+end

data/lib/simple_metrics/bucket.rb

@@ -5,39 +5,68 @@ module SimpleMetrics
     class << self
 
       def all
-        @@all ||= SimpleMetrics.config.buckets.map { |r| Bucket.new(r) }
+        @@all ||= SimpleMetrics.buckets_config.map { |r| Bucket.new(r) }
       end
 
       def first
         all.first
       end
+      alias :finest :first
 
       def [](index)
         all[index]
       end
 
-      def for_time(time)
-        case time
-        when 'minute'
-          self[0]
-        when 'hour'
-          self[1]
-        when 'day'
-          self[2]
-        when 'week'
-          self[3]
+      def coarse_buckets
+        Bucket.all.sort_by! { |r| r.seconds }[1..-1]
+      end
+
+      def flush_data_points(data_points)
+        return if data_points.empty?
+        SimpleMetrics.logger.info "#{Time.now} Flushing #{data_points.count} counters to MongoDB"
+
+        ts = Time.now.utc.to_i
+        bucket = Bucket.first
+        data_points.group_by { |data| data.name }.each_pair do |name,dps|
+          data = DataPoint.aggregate(dps)
+          bucket.save(data, ts)
+        end
+
+        self.aggregate_all(ts)
+      end
+
+      def aggregate_all(ts)
+        ts_bucket = self.first.ts_bucket(ts)
+
+        coarse_buckets.each do |bucket|
+          current_ts = bucket.ts_bucket(ts_bucket)
+          previous_ts = bucket.previous_ts_bucket(ts_bucket)
+          SimpleMetrics.logger.debug "Aggregating #{bucket.name} #{previous_ts}....#{current_ts} (#{humanized_timestamp(previous_ts)}..#{humanized_timestamp(current_ts)})"
+
+          unless bucket.stats_exist_in_previous_ts?(previous_ts)
+            data_points = self.first.find_all_in_ts_range(previous_ts, current_ts)
+            data_points.group_by { |data| data.name }.each_pair do |name,dps|
+              data = DataPoint.aggregate(dps)
+              bucket.save(data, previous_ts)
+            end
+          end
         end
       end
 
+      private
+
+      def humanized_timestamp(ts)
+        Time.at(ts).utc
+      end
     end
 
     attr_reader :name, :capped
 
     def initialize(attributes)
-      @name    = attributes.fetch(:name)
-      @seconds = attributes.fetch(:seconds)
-      @capped  = attributes.fetch(:capped)
-      @size    = attributes.fetch(:size)
+      @name    = attributes[:name]
+      @seconds = attributes[:seconds]
+      @capped  = attributes[:capped]
+      @size    = attributes[:size]
     end
 
     def seconds
@@ -49,7 +78,7 @@ module SimpleMetrics
     end
 
     def ts_bucket(ts)
-      (ts / seconds) * seconds
+      ts / seconds * seconds
     end
 
     def next_ts_bucket(ts)
@@ -60,33 +89,64 @@ module SimpleMetrics
       ts_bucket(ts) - seconds
     end
 
-    # TODO: only used in tests, do we need it?
-    def find_all_at_ts(ts)
-      repository.find_all_at_ts(ts_bucket(ts))
+    def find(id)
+      mongo_result = mongo_coll.find_one({ :_id => id })
+      DataPoint.create_from_db(mongo_result)
     end
 
-    def find_data_point_at_ts(ts, name)
-      repository.find_data_point_at_ts(ts_bucket(ts), name)
+    def find_all_by_name(name)
+      mongo_result = mongo_coll.find({ :name => name }).to_a
+      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
+    end
+
+    def find_all_in_ts(ts)
+      mongo_result = mongo_coll.find({ :ts => ts_bucket(ts) }).to_a
+      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
+    end
+
+    def find_all_in_ts_by_name(ts, name)
+      mongo_result = mongo_coll.find({ :ts => ts_bucket(ts), :name => name }).to_a
+      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
+    end
+
+    def find_all_in_ts_range(from, to)
+      mongo_result = mongo_coll.find({ :ts => { "$gte" => from, "$lte" => to }}).to_a
+      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
     end
 
     def find_all_in_ts_range_by_name(from, to, name)
-      repository.find_all_in_ts_range_by_name(from, to, name)
+      mongo_result = mongo_coll.find({ :name => name, :ts => { "$gte" => from, "$lte" => to }}).to_a
+      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
     end
 
     def find_all_in_ts_range_by_wildcard(from, to, target)
-      repository.find_all_in_ts_range_by_wildcard(from, to, target)
+      target = target.gsub('.', '\.')
+      target = target.gsub('*', '.*')
+      mongo_result = mongo_coll.find({ :name => /#{target}/, :ts => { "$gte" => from, "$lte" => to } }).to_a
+      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
     end
 
-    def save(dp, ts)
-      dp.ts    = ts_bucket(ts)
-      dp.sum   = dp.value
-      dp.total = 1
-      repository.save(dp)
+    def find_all_in_ts_range_by_regexp(from, to, target)
+      mongo_result = mongo_coll.find({ :name => /#{target}/, :ts => { "$gte" => from, "$lte" => to } }).to_a
+      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
     end
 
-    def update(dp, ts)
-      dp.ts = ts_bucket(ts)
-      repository.update(dp, ts)
+    def stats_exist_in_previous_ts?(ts)
+      mongo_coll.find({ :ts => ts }).count > 0
+    end
+
+    def find_all_distinct_names
+      mongo_coll.distinct(:name).to_a
+    end
+
+    def save(stats, ts)
+      stats.ts = ts_bucket(ts)
+      result = mongo_coll.insert(stats.attributes)
+      SimpleMetrics.logger.debug "SERVER: MongoDB - insert in #{name}: #{stats.inspect}, result: #{result}"
+    end
+
+    def mongo_coll
+      Mongo.collection(name)
     end
 
     def capped?
@@ -96,28 +156,31 @@ module SimpleMetrics
     def fill_gaps(from, to, query_result)
       return query_result if query_result.nil? || query_result.size == 0
       
-      existing_ts_entries =  query_result.inject({}) { |result, dp| result[dp.ts] = dp; result }
+      tmp_hash = DataPoint.ts_hash(query_result)
       dp_template = query_result.first
 
       result = []
-      each_ts(from, to) do |ts_bucket|
-        dp = existing_ts_entries[ts_bucket] || DataPoint::Base.new(:name => dp_template.name, :ts => ts_bucket)
-        result << dp
+      each_ts(from, to) do |current_bucket_ts|
+        result <<  
+          if tmp_hash.key?(current_bucket_ts)
+            tmp_hash[current_bucket_ts]
+          else
+            dp       = dp_template.dup
+            dp.value = nil
+            dp.ts    = current_bucket_ts
+            dp
+          end
       end
       result
     end
 
     private
 
-    def repository
-      DataPointRepository.for_retention(name)
-    end
-
     def each_ts(from, to)
-      ts_bucket = ts_bucket(from)
-      while (ts_bucket <= ts_bucket(to))
-        yield(ts_bucket)
-        ts_bucket = ts_bucket + seconds
+      current_bucket_ts = ts_bucket(from)
+      while (current_bucket_ts <= ts_bucket(to))
+        yield(current_bucket_ts)
+        current_bucket_ts = current_bucket_ts + seconds
       end
     end