simple_metrics 0.4.0 → 0.4.2

data/.travis.yml

@@ -0,0 +1,3 @@
+rvm:
+  - 1.9.2
+  - 1.9.3

data/README.markdown

@@ -9,78 +9,26 @@ Technically speaking it provides a simple UDP interface to send data to an Event
 
 SimpleMetrics is written in Ruby and packaged as a gem.
 
-The current version is considered ALPHA.
+The current version is considered ALPHA. 
 
-SimpleMetrics Client
---------------------
-
-Commandline client:
-
-Send a count of 5 for data point "module.test1":
-
-		simple_metrics_client module.test1 -counter 5
-
-Send a timing of 100ms:
-
-		simple_metrics_client module.test1 -timing 100
-
-doing the same, but since we expect a lot of calls we sample the data (10%):
-
-		simple_metrics_client module.test1 -timing 100 --sample_rate 0.1
-
-more info:
-	
-		simple_metrics_client --help
-
-Ruby client API
----------------
-
-Initialize client:
-
-		client = SimpleMetrics::Client.new("localhost")
-
-sends "com.example.test1:1|c" via UDP:
-
-		client.increment("com.example.test1")
-
-sends "com.example.test1:-1|c":
-
-		client.decrement("com.example.test1")
-
-sends "com.example.test1:5|c" (a counter with a relative value of 5):
-
-		client.count("com.example.test1", 5)
-
-sends "com.example.test1:5|c|@0.1" with a sample rate of 10%:
-
-		client.count("com.example.test1", 5, 0.1)
-
-sends "com.example.test1:5|g" (meaning gauge, an absolute value of 5):
-
-		client.count("com.example.test1", 5)
-
-sends "com.example.test1:100|ms":
-
-		client.timing("com.example.test1")
-
-More examples in the examples/ directory.
+There is a rails app provided to visualize the graphs in dashboards.
 
 SimpleMetrics Server
 --------------------
 
 We provide a simple commandline wrapper using daemons gem (http://daemons.rubyforge.org/).
 
-Start Server as background daemond:
+Start Server as background daemon:
 
-		simple_metrics_server start
+    simple_metrics_server start
 
 Start in foreground:
 
-		simple_metrics_server start -t
+    simple_metrics_server start -t
 
 Show Help:
 
-		simple_metrics_server --help
+    simple_metrics_server --help
 
 Round Robin Database Principles in MongoDB
 ------------------------------------------
@@ -91,7 +39,7 @@ We use 4 collections in MongoDB each with more coarse timestamp buckets:
 * 10 min
 * 1 day
 
-The 10s and 1m collections are capped collections and have a fixed size. The other will store the data as long as we have sufficient disc space.
+The 10sec and 1min collections are capped collections and have a fixed size. The others will store the data as long as there is sufficient disc space.
 
 How can we map these times to graphs?
 

data/Rakefile

@@ -2,7 +2,24 @@ require "bundler/gem_tasks"
 require 'rake'
 require 'rspec/core/rake_task'
 
+require 'simple_metrics'
+
 RSpec::Core::RakeTask.new 
 task :default  => :spec
 task :test  => :spec
 
+namespace :simple_metrics do
+
+  desc "Ensure collections and index exist"
+  task :ensure_collections_exist do
+    SimpleMetrics.logger = Logger.new($stdout)
+    SimpleMetrics::DataPointRepository.ensure_collections_exist
+  end
+
+  desc "Truncate all collections"
+  task :truncate_collections do
+    SimpleMetrics.logger = Logger.new($stdout)
+    SimpleMetrics::DataPointRepository.truncate_collections
+  end
+
+end

data/bin/populate

@@ -6,20 +6,33 @@ require "bundler/setup"
 require 'optparse'
 require "simple_metrics"
 
-ts = Time.now.to_i
+SimpleMetrics.logger = Logger.new("/dev/null")
+SimpleMetrics::DataPointRepository.truncate_collections
+SimpleMetrics::DataPointRepository.ensure_collections_exist
 
-name = "test.post.clicks2"
+bucket = SimpleMetrics::Bucket.first
 
-SimpleMetrics::Mongo.ensure_collections_exist
+name = ENV['NAME'] || "test.page.visits.1"
+now = Time.now.to_i
+minute = 60
+hour   = minute * 60
 
-SimpleMetrics::Bucket.all.each do |bucket|
-	ts_bucket = bucket.ts_bucket(ts)
+def create_dps(name)
+  previous_value = 20
+  (1..1).inject([]) do |result, index|
+    value = previous_value+rand(20)
+    result << SimpleMetrics::DataPoint::Counter.new(:name => name, :value => value)
+    previous_value = value
+    result
+  end
+end
 
-	previous_value = rand(300)
-	(1..100).each do |index|
-		ts = ts_bucket - (bucket.seconds*index)
-		value = previous_value+rand(20)
-		data_point = SimpleMetrics::DataPoint.create_counter(:name => name, :value => value)
-		bucket.save(data_point, ts)
-	end
+counter = 1
+current = now - 1 * hour
+while (current < now)
+  dps = create_dps(name)
+  puts "flush data for #{Time.at(current)}, #{counter}"
+  SimpleMetrics::Importer.flush_data_points(dps, current)
+  current += 10
+  counter += 1
 end

data/bin/populate_for_demo

@@ -0,0 +1,65 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "bundler/setup"
+
+require 'optparse'
+require "simple_metrics"
+
+SimpleMetrics.logger = Logger.new("/dev/null")
+SimpleMetrics::DataPointRepository.truncate_collections
+SimpleMetrics::DataPointRepository.ensure_collections_exist
+SimpleMetrics::MetricRepository.truncate_collections
+
+def create_dps(name)
+  previous_value = 20
+  (1..1).inject([]) do |result, index|
+    value = previous_value+rand(20)
+    result << SimpleMetrics::DataPoint::Counter.new(:name => name, :value => value)
+    previous_value = value
+    result
+  end
+end
+
+def flush_data_points(name)
+  #bucket = SimpleMetrics::Bucket.first
+  now = Time.now.to_i
+  minute = 60
+  hour   = minute * 60
+
+  counter = 1
+  current = now - 1 * hour
+  while (current < now)
+    dps = create_dps(name)
+    puts "flush data for #{Time.at(current)}, #{counter}"
+    SimpleMetrics::Importer.flush_data_points(dps, current)
+    current += 10
+    counter += 1
+  end
+end
+
+
+name1 =  "test.page.visits"
+flush_data_points(name1)
+name2 =  "test.invoices.send"
+flush_data_points(name2)
+
+metric1 = SimpleMetrics::MetricRepository.find_one_by_name(name1)
+metric2 = SimpleMetrics::MetricRepository.find_one_by_name(name2)
+
+SimpleMetrics::InstrumentRepository.truncate_collections
+instrument1 = SimpleMetrics::Instrument.new(:name => "Example Instrument 1", :metrics => [{ :name => metric1.name }, { :name => metric2.name}])
+instrument2 = SimpleMetrics::Instrument.new(:name => "Example Instrument 2", :metrics => [{ :name => metric1.name} ])
+instrument1.id = SimpleMetrics::InstrumentRepository.save(instrument1)
+instrument2.id = SimpleMetrics::InstrumentRepository.save(instrument2)
+
+puts instrument1.inspect
+puts instrument2.inspect
+10.times do |i|
+  instrument = SimpleMetrics::Instrument.new(:name => "Example Instrument Randmon #{i}", :metrics => [{ :name => metric1.name}])
+  SimpleMetrics::InstrumentRepository.save(instrument)
+end
+
+SimpleMetrics::DashboardRepository.truncate_collections
+SimpleMetrics::DashboardRepository.save(SimpleMetrics::Dashboard.new(:name => "Example Dashboard 1", :instruments => [instrument1.id.to_s, instrument2.id.to_s]))
+SimpleMetrics::DashboardRepository.save(SimpleMetrics::Dashboard.new(:name => "Example Dashboard 2", :instruments => [instrument1.id.to_s]))

data/bin/simple_metrics_server

@@ -1,18 +1,19 @@
 #!/usr/bin/env ruby
 
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
 require "rubygems"
-require "bundler/setup"
 require "simple_metrics"
 require "daemons"
 
 options = {
 	:backtrace  => true,
 	:log_output => true,
-	:dir_mode   => :script
+	:dir_mode   => :script,
 }
 
 Daemons.run_proc("simple_metrics", options) do
-	SimpleMetrics::Server.new.start
+	SimpleMetrics::UDPServer.new.start
 end
 
 

data/config.ru

@@ -0,0 +1,13 @@
+$LOAD_PATH.unshift ::File.expand_path(::File.dirname(__FILE__) + '/lib')
+
+require "rubygems"
+require "simple_metrics"
+require "simple_metrics/app"
+
+map "/assets" do
+  run SimpleMetrics::App.sprockets
+end
+
+map "/" do
+  run SimpleMetrics::App
+end

data/default_config.yml

@@ -0,0 +1,34 @@
+server:
+  host: 'localhost'
+  port: 8125
+  flush_interval: 10
+
+db:
+  host: 'localhost'
+  port: 27017
+  prefix: 'development'
+  options:
+    pool_size: 5
+    timeout: 5
+
+web:
+  host: 'localhost'
+  port: 5678
+
+buckets: 
+    - name: 'stats_per_10s'
+      seconds: 10
+      capped: true
+      size: 180_000 # 6*30 = 180 (30 min ttl), 180 * 100 = 18.000 (100 data points), 18.000 * 10 (bytes each) = 180.000
+    - name: 'stats_per_1min'
+      seconds: 60
+      capped: true
+      size: 120_000 # 60*3 = 180 (3h ttl), 120 * 100 = 18.000 (100 data points), 18.000 * 10 (bytes each) = 180.000
+    - name: 'stats_per_10min'
+      seconds: 600
+      size: 0 
+      capped: false
+    - name: 'stats_per_day'
+      seconds: 86400 # 600*6*24
+      size: 0 
+      capped: false

data/lib/simple_metrics/bucket.rb

@@ -5,68 +5,39 @@ module SimpleMetrics
     class << self
 
       def all
-        @@all ||= SimpleMetrics.buckets_config.map { |r| Bucket.new(r) }
+        @@all ||= SimpleMetrics.config.buckets.map { |r| Bucket.new(r) }
       end
 
       def first
         all.first
       end
-      alias :finest :first
 
       def [](index)
         all[index]
       end
 
-      def coarse_buckets
-        Bucket.all.sort_by! { |r| r.seconds }[1..-1]
-      end
-
-      def flush_data_points(data_points)
-        return if data_points.empty?
-        SimpleMetrics.logger.info "#{Time.now} Flushing #{data_points.count} counters to MongoDB"
-
-        ts = Time.now.utc.to_i
-        bucket = Bucket.first
-        data_points.group_by { |data| data.name }.each_pair do |name,dps|
-          data = DataPoint.aggregate(dps)
-          bucket.save(data, ts)
-        end
-
-        self.aggregate_all(ts)
-      end
-
-      def aggregate_all(ts)
-        ts_bucket = self.first.ts_bucket(ts)
-
-        coarse_buckets.each do |bucket|
-          current_ts = bucket.ts_bucket(ts_bucket)
-          previous_ts = bucket.previous_ts_bucket(ts_bucket)
-          SimpleMetrics.logger.debug "Aggregating #{bucket.name} #{previous_ts}....#{current_ts} (#{humanized_timestamp(previous_ts)}..#{humanized_timestamp(current_ts)})"
-
-          unless bucket.stats_exist_in_previous_ts?(previous_ts)
-            data_points = self.first.find_all_in_ts_range(previous_ts, current_ts)
-            data_points.group_by { |data| data.name }.each_pair do |name,dps|
-              data = DataPoint.aggregate(dps)
-              bucket.save(data, previous_ts)
-            end
-          end
+      def for_time(time)
+        case time
+        when 'minute'
+          self[0]
+        when 'hour'
+          self[1]
+        when 'day'
+          self[2]
+        when 'week'
+          self[3]
         end
       end
 
-      private
-
-      def humanized_timestamp(ts)
-        Time.at(ts).utc
-      end
     end
 
     attr_reader :name, :capped
 
     def initialize(attributes)
-      @name    = attributes[:name]
-      @seconds = attributes[:seconds]
-      @capped  = attributes[:capped]
-      @size    = attributes[:size]
+      @name    = attributes.fetch(:name)
+      @seconds = attributes.fetch(:seconds)
+      @capped  = attributes.fetch(:capped)
+      @size    = attributes.fetch(:size)
     end
 
     def seconds
@@ -78,7 +49,7 @@ module SimpleMetrics
     end
 
     def ts_bucket(ts)
-      ts / seconds * seconds
+      (ts / seconds) * seconds
     end
 
     def next_ts_bucket(ts)
@@ -89,64 +60,33 @@ module SimpleMetrics
       ts_bucket(ts) - seconds
     end
 
-    def find(id)
-      mongo_result = mongo_coll.find_one({ :_id => id })
-      DataPoint.create_from_db(mongo_result)
+    # TODO: only used in tests, do we need it?
+    def find_all_at_ts(ts)
+      repository.find_all_at_ts(ts_bucket(ts))
     end
 
-    def find_all_by_name(name)
-      mongo_result = mongo_coll.find({ :name => name }).to_a
-      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
-    end
-
-    def find_all_in_ts(ts)
-      mongo_result = mongo_coll.find({ :ts => ts_bucket(ts) }).to_a
-      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
-    end
-
-    def find_all_in_ts_by_name(ts, name)
-      mongo_result = mongo_coll.find({ :ts => ts_bucket(ts), :name => name }).to_a
-      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
-    end
-
-    def find_all_in_ts_range(from, to)
-      mongo_result = mongo_coll.find({ :ts => { "$gte" => from, "$lte" => to }}).to_a
-      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
+    def find_data_point_at_ts(ts, name)
+      repository.find_data_point_at_ts(ts_bucket(ts), name)
     end
 
     def find_all_in_ts_range_by_name(from, to, name)
-      mongo_result = mongo_coll.find({ :name => name, :ts => { "$gte" => from, "$lte" => to }}).to_a
-      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
+      repository.find_all_in_ts_range_by_name(from, to, name)
     end
 
     def find_all_in_ts_range_by_wildcard(from, to, target)
-      target = target.gsub('.', '\.')
-      target = target.gsub('*', '.*')
-      mongo_result = mongo_coll.find({ :name => /#{target}/, :ts => { "$gte" => from, "$lte" => to } }).to_a
-      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
+      repository.find_all_in_ts_range_by_wildcard(from, to, target)
     end
 
-    def find_all_in_ts_range_by_regexp(from, to, target)
-      mongo_result = mongo_coll.find({ :name => /#{target}/, :ts => { "$gte" => from, "$lte" => to } }).to_a
-      mongo_result.inject([]) { |result, a| result << DataPoint.create_from_db(a) }
+    def save(dp, ts)
+      dp.ts    = ts_bucket(ts)
+      dp.sum   = dp.value
+      dp.total = 1
+      repository.save(dp)
     end
 
-    def stats_exist_in_previous_ts?(ts)
-      mongo_coll.find({ :ts => ts }).count > 0
-    end
-
-    def find_all_distinct_names
-      mongo_coll.distinct(:name).to_a
-    end
-
-    def save(stats, ts)
-      stats.ts = ts_bucket(ts)
-      result = mongo_coll.insert(stats.attributes)
-      SimpleMetrics.logger.debug "SERVER: MongoDB - insert in #{name}: #{stats.inspect}, result: #{result}"
-    end
-
-    def mongo_coll
-      Mongo.collection(name)
+    def update(dp, ts)
+      dp.ts = ts_bucket(ts)
+      repository.update(dp, ts)
     end
 
     def capped?
@@ -156,31 +96,28 @@ module SimpleMetrics
     def fill_gaps(from, to, query_result)
       return query_result if query_result.nil? || query_result.size == 0
       
-      tmp_hash = DataPoint.ts_hash(query_result)
+      existing_ts_entries =  query_result.inject({}) { |result, dp| result[dp.ts] = dp; result }
       dp_template = query_result.first
 
       result = []
-      each_ts(from, to) do |current_bucket_ts|
-        result <<  
-          if tmp_hash.key?(current_bucket_ts)
-            tmp_hash[current_bucket_ts]
-          else
-            dp       = dp_template.dup
-            dp.value = nil
-            dp.ts    = current_bucket_ts
-            dp
-          end
+      each_ts(from, to) do |ts_bucket|
+        dp = existing_ts_entries[ts_bucket] || DataPoint::Base.new(:name => dp_template.name, :ts => ts_bucket)
+        result << dp
       end
       result
     end
 
     private
 
+    def repository
+      DataPointRepository.for_retention(name)
+    end
+
     def each_ts(from, to)
-      current_bucket_ts = ts_bucket(from)
-      while (current_bucket_ts <= ts_bucket(to))
-        yield(current_bucket_ts)
-        current_bucket_ts = current_bucket_ts + seconds
+      ts_bucket = ts_bucket(from)
+      while (ts_bucket <= ts_bucket(to))
+        yield(ts_bucket)
+        ts_bucket = ts_bucket + seconds
       end
     end
 

data/lib/simple_metrics/configuration.rb

@@ -0,0 +1,82 @@
+require 'yaml'
+
+module SimpleMetrics
+  class Configuration
+
+    attr_reader :config
+
+    def initialize(hash = {}, &block)
+      @config = load_defaults.merge(hash.symbolize_keys)
+    end
+
+    def configure(hash = {}, &block)
+      yield self if block_given?
+      self
+    end
+
+    def db
+      @db ||= config.fetch(:db).symbolize_keys
+    end
+
+    def db=(db)
+      @db = db
+    end
+
+    def buckets
+      @buckets ||= begin
+        tmp = config.fetch(:buckets)
+        tmp.map { |b| b.symbolize_keys }
+      end
+    end
+
+    def buckets=(buckets)
+      @buckets = buckets
+    end
+
+    def server
+      @server ||= config.fetch(:server).symbolize_keys
+    end
+
+    def server=(server)
+      @server = server
+    end
+
+    def web
+      @web ||= config.fetch(:web).symbolize_keys
+    end
+
+    def web=(web)
+      @web = web
+    end
+
+    private
+
+    def load_defaults
+      @config = load_config
+    rescue Errno::ENOENT # not found error
+      logger.info "Creating initial config file: #{config_file}"
+      FileUtils.cp(default_config_file, config_file)
+      @config = load_config
+    end
+
+    def config_file
+      File.expand_path('~/.simple_metrics.conf')
+    end
+
+    def default_config_file
+      File.expand_path('../../../default_config.yml', __FILE__)
+    end
+
+    def load_config
+      YAML.load_file(config_file).symbolize_keys
+    rescue ArgumentError => e
+      logger.error "Error parsing config file: #{e}"
+    rescue IOError => e
+      logger.error "Error reading config file: #{e}"
+    end
+
+    def logger
+      SimpleMetrics.logger
+    end
+  end
+end

data/lib/simple_metrics/dashboard.rb

@@ -0,0 +1,29 @@
+module SimpleMetrics
+  class Dashboard
+
+    attr_reader :id, :created_at, :updated_at
+    attr_accessor :name, :instruments
+
+    def initialize(attributes)
+      @id          = attributes[:id]
+      @name        = attributes[:name]
+      @created_at  = attributes[:created_at]
+      @updated_at  = attributes[:updated_at]
+      @instruments = attributes[:instruments] || []
+    end
+
+    def attributes
+      { 
+        :id          => @id.to_s, # convert bson id to str
+        :name        => @name,
+        :instruments => @instruments,
+        :created_at  => @created_at,
+        :updated_at  => @updated_at
+      }
+    end
+
+    def to_s
+      attributes.to_s
+    end
+  end
+end