simple_metrics 0.2.3 → 0.3.2

data/.travis.yml

@@ -0,0 +1,3 @@
+rvm:
+  - 1.9.2
+  - 1.9.3

data/README.markdown

@@ -11,76 +11,39 @@ SimpleMetrics is written in Ruby and packaged as a gem.
 
 The current version is considered ALPHA.
 
-SimpleMetrics Client
+SimpleMetrics Server
 --------------------
 
-Commandline client:
-
-Send a count of 5 for data point "module.test1":
-
-		simple_metrics_client module.test1 -counter 5
-
-Send a timing of 100ms:
-
-		simple_metrics_client module.test1 -timing 100
-
-doing the same, but since we expect a lot of calls we sample the data (10%):
-
-		simple_metrics_client module.test1 -timing 100 --sample_rate 0.1
-
-more info:
-	
-		simple_metrics_client --help
-
-Ruby client API
----------------
-
-Initialize client:
-
-		client = SimpleMetrics::Client.new("localhost")
-
-sends "com.example.test1:1|c" via UDP:
-
-		client.increment("com.example.test1")
-
-sends "com.example.test1:-1|c":
-
-		client.decrement("com.example.test1")
-
-sends "com.example.test1:5|c" (a counter with a relative value of 5):
-
-		client.count("com.example.test1", 5)
-
-sends "com.example.test1:5|c|@0.1" with a sample rate of 10%:
+We provide a simple commandline wrapper using daemons gem (http://daemons.rubyforge.org/).
 
-		client.count("com.example.test1", 5, 0.1)
+Start Server as background daemon:
 
-sends "com.example.test1:5|g" (meaning gauge, an absolute value of 5):
+    simple_metrics_server start
 
-		client.count("com.example.test1", 5)
+Start in foreground:
 
-sends "com.example.test1:100|ms":
+    simple_metrics_server start -t
 
-		client.timing("com.example.test1")
+Show Help:
 
-More examples in the examples/ directory.
+    simple_metrics_server --help
 
-SimpleMetrics Server
---------------------
+SimpleMetrics Web App
+-----------------
 
-We provide a simple commandline wrapper using daemons gem (http://daemons.rubyforge.org/).
+A small Sinatra app is provided using the vegas gem (https://github.com/quirkey/vegas).
 
-Start Server as background daemond:
+Start web app as background daemon:
 
-		simple_metrics_server start
+    simple_metrics_web
 
 Start in foreground:
 
-		simple_metrics_server start -t
+    simple_metrics_web -F
 
 Show Help:
 
-		simple_metrics_server --help
+    simple_metrics_web --help
 
 Round Robin Database Principles in MongoDB
 ------------------------------------------
@@ -91,7 +54,7 @@ We use 4 collections in MongoDB each with more coarse timestamp buckets:
 * 10 min
 * 1 day
 
-The 10s and 1m collections are capped collections and have a fixed size. The other will store the data as long as we have sufficient disc space.
+The 10sec and 1min collections are capped collections and have a fixed size. The others will store the data as long as there is sufficient disc space.
 
 How can we map these times to graphs?
 

data/Rakefile

@@ -2,7 +2,24 @@ require "bundler/gem_tasks"
 require 'rake'
 require 'rspec/core/rake_task'
 
+require 'simple_metrics'
+
 RSpec::Core::RakeTask.new 
 task :default  => :spec
 task :test  => :spec
 
+namespace :simple_metrics do
+
+  desc "Ensure collections and index exist"
+  task :ensure_collections_exist do
+    SimpleMetrics.logger = Logger.new($stdout)
+    SimpleMetrics::DataPointRepository.ensure_collections_exist
+  end
+
+  desc "Truncate all collections"
+  task :truncate_collections do
+    SimpleMetrics.logger = Logger.new($stdout)
+    SimpleMetrics::DataPointRepository.truncate_collections
+  end
+
+end

data/bin/populate

@@ -6,20 +6,33 @@ require "bundler/setup"
 require 'optparse'
 require "simple_metrics"
 
-ts = Time.now.to_i
+SimpleMetrics.logger = Logger.new("/dev/null")
+SimpleMetrics::DataPointRepository.truncate_collections
+SimpleMetrics::DataPointRepository.ensure_collections_exist
 
-name = "test.post.clicks2"
+bucket = SimpleMetrics::Bucket.first
 
-SimpleMetrics::Mongo.ensure_collections_exist
+name = ENV['NAME'] || "test.page.visits.1"
+now = Time.now.to_i
+minute = 60
+hour   = minute * 60
 
-SimpleMetrics::Bucket.all.each do |bucket|
-	ts_bucket = bucket.ts_bucket(ts)
+def create_dps(name)
+  previous_value = 20
+  (1..1).inject([]) do |result, index|
+    value = previous_value+rand(20)
+    result << SimpleMetrics::DataPoint.create_counter(:name => name, :value => value)
+    previous_value = value
+    result
+  end
+end
 
-	previous_value = rand(300)
-	(1..100).each do |index|
-		ts = ts_bucket - (bucket.seconds*index)
-		value = previous_value+rand(20)
-		data_point = SimpleMetrics::DataPoint.create_counter(:name => name, :value => value)
-		bucket.save(data_point, ts)
-	end
-end
+counter = 1
+current = now - 1 * hour
+while (current < now)
+  dps = create_dps(name)
+  puts "flush data for #{Time.at(current)}, #{counter}"
+  SimpleMetrics::Bucket.flush_data_points(dps, current)
+  current += 10
+  counter += 1
+end

data/bin/simple_metrics_server

@@ -1,18 +1,19 @@
 #!/usr/bin/env ruby
 
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
 require "rubygems"
-require "bundler/setup"
 require "simple_metrics"
 require "daemons"
 
 options = {
 	:backtrace  => true,
 	:log_output => true,
-	:dir_mode   => :script
+	:dir_mode   => :script,
 }
 
 Daemons.run_proc("simple_metrics", options) do
-	SimpleMetrics::Server.new.start
+	SimpleMetrics::UDPServer.new.start
 end
 
 

data/bin/simple_metrics_web

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift File.expand_path(File.dirname(__FILE__) + '/../lib')
+
+require "rubygems"
+require 'vegas'
+require "simple_metrics"
+
+Vegas::Runner.new(SimpleMetrics::App, 'simple_metrics_web') do |runner, opts, app|
+  # add more options here
+end

data/config.ru

@@ -0,0 +1,6 @@
+$LOAD_PATH.unshift ::File.expand_path(::File.dirname(__FILE__) + '/lib')
+
+require "rubygems"
+require "simple_metrics"
+
+run SimpleMetrics::App.new

data/default_config.yml

@@ -0,0 +1,34 @@
+server:
+  host: 'localhost'
+  port: 8125
+  flush_interval: 10
+
+db:
+  host: 'localhost'
+  port: 27017
+  prefix: 'development'
+  options:
+    pool_size: 5
+    timeout: 5
+
+web:
+  host: 'localhost'
+  port: 5678
+
+buckets: 
+    - name: 'stats_per_10s'
+      seconds: 10
+      capped: true
+      size: 180_000 # 6*30 = 180 (30 min ttl), 180 * 100 = 18.000 (100 data points), 18.000 * 10 (bytes each) = 180.000
+    - name: 'stats_per_1min'
+      seconds: 60
+      capped: true
+      size: 120_000 # 60*3 = 180 (3h ttl), 120 * 100 = 18.000 (100 data points), 18.000 * 10 (bytes each) = 180.000
+    - name: 'stats_per_10min'
+      seconds: 600
+      size: 0 
+      capped: false
+    - name: 'stats_per_day'
+      seconds: 86400 # 600*6*24
+      size: 0 
+      capped: false

data/lib/simple_metrics/app.rb

@@ -0,0 +1,136 @@
+require "sinatra"
+require "erubis"
+require "json"
+
+if defined? Encoding
+  Encoding.default_external = Encoding::UTF_8
+end
+
+module SimpleMetrics
+  class App < Sinatra::Base
+
+    set :views, ::File.expand_path('../views', __FILE__)  
+    set :public_folder, File.expand_path('../public', __FILE__)
+
+    helpers do
+      def graph_title(time)
+        case time
+        when "minute"
+          "Minute"
+        when "hour"
+          "Hour"
+        when "day"
+          "Day"
+        when "week"
+          "Week"
+        end
+      end
+    end
+
+    get "/api/metrics" do
+      content_type :json
+      metrics = SimpleMetrics::MetricRepository.find_all
+      metrics.inject([]) { |result, m| result << m.attributes }.to_json
+    end
+
+    get "/api/metrics/:id" do
+      content_type :json
+      metric = SimpleMetrics::MetricRepository.find_one(to_bson_id(params[:id]))
+      metric.attributes.to_json
+    end
+
+    get "/api/graph" do
+      content_type :json
+      from    = (params[:from]  || Time.now).to_i
+      time    = params[:time]   || 'minute'
+      targets = params[:targets]
+      data_points = prepare_data_points(from, time, *targets)
+      data_points.to_json
+    end
+
+    get "/*" do
+      erb :index
+    end
+
+    # get "/metrics" do
+    #   erb :index
+    # end
+
+    # get "/metrics/:id" do
+    #   erb :index
+    # end
+
+    # get "/metric" do
+    #   @from = (params[:from]  || Time.now).to_i
+    #   erb :show
+    # end
+
+    # get "/graph" do
+    #   @from    = (params[:from]  || Time.now).to_i
+    #   @time    = params[:time]   || 'minute'
+    #   @targets = params[:target]
+    #   @data_points = prepare_data_points(@from, @time, *@targets)
+    #   @series = @data_points
+    #   erb :graph, :layout => false
+    # end
+
+    private
+
+    # params[:id]
+    def to_bson_id(id) 
+      BSON::ObjectId.from_string(id) 
+    end 
+
+    def prepare_data_points(from, time, *targets)
+      to = from - time_range(time)
+      result = SimpleMetrics::Graph.query_all(bucket(time), to, from, *targets)
+      result.map do |data_point|
+        { :name => data_point.first, :data => data_point.last.map { |p| { :x => p[:ts], :y => p[:value] || 0 } } }
+      end
+    end
+
+    def one_minute
+      60
+    end
+
+    def one_hour
+      one_minute * 60
+    end
+
+    def one_day
+      one_hour * 24
+    end
+
+    def one_week
+      one_day * 7
+    end
+
+    def time_range(time)
+      case time
+      when 'minute'
+        5*one_minute
+      when 'hour'
+        one_hour
+      when 'day'
+        one_day
+      when 'week'
+        one_week
+      else 
+        raise "Unknown time param: #{time}"
+      end
+    end
+
+    def bucket(time)
+      case time
+      when 'minute'
+        SimpleMetrics::Bucket[0]
+      when 'hour'
+        SimpleMetrics::Bucket[1]
+      when 'day'
+        SimpleMetrics::Bucket[2]
+      when 'week'
+        SimpleMetrics::Bucket[3]
+      end
+    end
+  end
+end

data/lib/simple_metrics/array_aggregation.rb

@@ -0,0 +1,84 @@
+module SimpleMetrics
+  module ArrayAggregation
+    extend self
+
+    class Base
+      def aggregate(dps, name = nil)
+        raise "subclass must implement me!"
+      end
+
+      private
+
+      def ts_hash_aggregated(data_points, &block)
+        tmp = {}
+        data_points.each do |dp|
+          if tmp.key?(dp.ts)
+            tmp[dp.ts] = block.call(tmp[dp.ts], dp.value)
+          else
+            tmp[dp.ts] = dp.value
+          end
+        end
+        tmp
+      end
+    end
+
+    class Counter < Base
+      def aggregate(dps, name = nil)
+        tmp_hash = ts_hash_aggregated(dps) do |value1, value2|
+          value1 + value2
+        end
+
+        result = []
+        tmp_hash.each_pair do |key, value|
+          result << SimpleMetrics::DataPoint::Counter.new(:name => name, :ts => key, :value => value)
+        end
+        result
+      end
+    end
+
+    class Gauge < Base
+      def aggregate(dps, name = nil)
+        tmp_hash = ts_hash_aggregated(dps) do |value1, value2|
+          (value1 + value2)/2
+        end
+
+        result = []
+        tmp_hash.each_pair do |key, value|
+          result << SimpleMetrics::DataPoint::Gauge.new(:name => name, :ts => key, :value => value)
+        end
+        result
+      end
+    end
+
+    class Timing < Base
+    end
+
+    class Event < Base
+    end
+
+    def aggregate(dps, name = nil)
+      raise SimpleMetrics::DataPoint::NonMatchingTypesError if has_non_matching_types?(dps)
+
+      dp = dps.first
+      strategy(dp).aggregate(dps, name)
+    end
+
+    private
+
+    def strategy(dp)
+      if dp.counter?
+        Counter.new
+      elsif dp.gauge?
+        Gauge.new
+      elsif dp.timing?
+        Timing.new
+      elsif dp.event?
+        Event.new
+      end
+    end
+
+    def has_non_matching_types?(dps)
+      dps.group_by { |dp| dp.type }.size != 1
+    end
+  end
+end