metriks 0.8.1 → 0.8.2

data/Gemfile

@@ -2,6 +2,10 @@ source :rubygems
 
 gemspec
 
+group :development do
+  gem 'perftools.rb'
+end
+
 group :test do
   gem 'turn'
 end

data/README.md

@@ -9,13 +9,12 @@ I find needs while developing [Papertrail](https://papertrailapp.com/).
 
 # Installing
 
-Everything is still in flux, so for the time being I have been installing
-the gem from git with bundler. If I get a request, I can definitely start
-releasing a gem.
+The API is still in flux, but you can add this to your project by installing
+the gem.
 
 To install, add this to your `Gemfile`:
 
-    gem 'metriks', :git => 'git://github.com/eric/metriks.git'
+    gem 'metriks'
 
 and re-run `bundle`.
 

data/benchmark/samplers.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+require 'benchmark'
+require 'metriks'
+
+
+def fib(n)
+  n < 2 ? n : fib(n-1) + fib(n-2)
+end
+
+uniform_timer     = Metriks::Timer.new(Metriks::Histogram.new_uniform)
+exponential_timer = Metriks::Timer.new(Metriks::Histogram.new_exponentially_decaying)
+
+fib_times = ARGV[0] ? ARGV[0].to_i : 10
+iter      = ARGV[1] ? ARGV[1].to_i : 100000
+
+puts "fib(#{fib_times}): #{iter} iterations"
+puts "-" * 50
+
+plain = Benchmark.realtime do
+  for i in 1..iter
+    fib(fib_times)
+  end
+end
+
+puts "%15s: %f secs %f secs/call" % [ 'plain', plain, plain / iter ]
+
+uniform = Benchmark.realtime do
+  for i in 1..iter
+    uniform_timer.time do
+      fib(fib_times)
+    end
+  end
+end
+
+puts "%15s: %f secs %f secs/call -- %.1f%% slower than plain (%f secs/call)" % [ 
+  'uniform', uniform, uniform / iter, 
+  (uniform - plain) / plain * 100 ,
+  (uniform - plain) / iter,
+]
+
+exponential = Benchmark.realtime do
+  for i in 1..iter
+    exponential_timer.time do
+      fib(fib_times)
+    end
+  end
+end
+
+puts "%15s: %f secs %f secs/call -- %.1f%% slower than plain (%f secs/call) -- %.1f%% slower than uniform (%f secs/call)" % [ 
+  'exponential', exponential, exponential / iter, 
+  (exponential - plain) / plain * 100 ,
+  (exponential - plain) / iter,
+  (exponential - uniform) / uniform * 100 ,
+  (exponential - uniform) / iter
+]
+
+

data/lib/metriks.rb

@@ -1,6 +1,6 @@
 
 module Metriks
-  VERSION = '0.8.1'
+  VERSION = '0.8.2'
 
   def self.get(name)
     Metriks::Registry.default.get(name)

data/lib/metriks/exponentially_decaying_sample.rb

@@ -0,0 +1,80 @@
+require 'atomic'
+require 'rbtree'
+require 'metriks/snapshot'
+
+module Metriks
+  class ExponentiallyDecayingSample
+    RESCALE_THRESHOLD = 60 * 60 # 1 hour
+
+    def initialize(reservoir_size, alpha)
+      @values = RBTree.new
+      @count = Atomic.new(0)
+      @next_scale_time = Atomic.new(0)
+      @alpha = alpha
+      @reservoir_size = reservoir_size
+      @mutex = Mutex.new
+      clear
+    end
+
+    def clear
+      @mutex.synchronize do
+        @values.clear
+        @count.value = 0
+        @next_scale_time.value = Time.now + RESCALE_THRESHOLD
+        @start_time = Time.now
+      end
+    end
+
+    def size
+      count = @count.value
+      count < @reservoir_size ? count : @reservoir_size
+    end
+
+    def snapshot
+      @mutex.synchronize do
+        Snapshot.new(@values.values)
+      end
+    end
+
+    def update(value, timestamp = Time.now)
+      @mutex.synchronize do
+        priority = Math.exp(timestamp - @start_time) / rand
+        new_count = @count.update { |v| v + 1 }
+        if new_count <= @reservoir_size
+          @values[priority] = value
+        else
+          first_priority = @values.first[0]
+          if first_priority < priority
+            unless @values[priority]
+              @values[priority] = value
+
+              until @values.delete(first_priority)
+                first_priority = @values.first[0]
+              end
+            end
+          end
+        end
+      end
+
+      now = Time.new
+      next_time = @next_scale_time.value
+      if now >= next_time
+        rescale(now, next_time)
+      end
+    end
+
+    def rescale(now, next_time)
+      if @next_scale_time.compare_and_swap(next_time, now + RESCALE_THRESHOLD)
+        @mutex.synchronize do
+          old_start_time = @start_time
+          @start_time = Time.now
+          keys = @values.keys
+          keys.each do |key|
+            value = @values.delete(key)
+            @values[key* Math.exp(-@alpha * (@start_time - old_start_time))] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/metriks/histogram.rb

@@ -1,5 +1,6 @@
 require 'atomic'
 require 'metriks/uniform_sample'
+require 'metriks/exponentially_decaying_sample'
 
 module Metriks
   class Histogram
@@ -10,6 +11,10 @@ module Metriks
       new(Metriks::UniformSample.new(DEFAULT_SAMPLE_SIZE))
     end
 
+    def self.new_exponentially_decaying
+      new(Metriks::ExponentiallyDecayingSample.new(DEFAULT_SAMPLE_SIZE, DEFAULT_ALPHA))
+    end
+
     def initialize(sample)
       @sample   = sample
       @count    = Atomic.new(0)
@@ -37,6 +42,10 @@ module Metriks
       update_variance(value)
     end
 
+    def snapshot
+      @sample.snapshot
+    end
+
     def count
       @count.value
     end

data/lib/metriks/reporter/logger.rb

@@ -51,12 +51,16 @@ module Metriks::Reporter
             :min, :max, :mean, :stddev,
             :one_minute_utilization, :five_minute_utilization,
             :fifteen_minute_utilization, :mean_utilization,
+          ], [
+            :median, :get_95th_percentile
           ]
         when Metriks::Timer
           log_metric name, 'timer', metric, [
             :count, :one_minute_rate, :five_minute_rate,
             :fifteen_minute_rate, :mean_rate,
             :min, :max, :mean, :stddev
+          ], [
+            :median, :get_95th_percentile
           ]
         end
       end
@@ -64,11 +68,12 @@ module Metriks::Reporter
 
     def extract_from_metric(metric, *keys)
       keys.flatten.collect do |key|
-        [ { key => metric.send(key) } ]
+        name = key.to_s.gsub(/^get_/, '')
+        [ { name => metric.send(key) } ]
       end
     end
 
-    def log_metric(name, type, metric, *keys)
+    def log_metric(name, type, metric, keys, snapshot_keys = [])
       message = []
 
       message << @prefix if @prefix
@@ -78,6 +83,11 @@ module Metriks::Reporter
       message << { :type => type }
       message += extract_from_metric(metric, keys)
 
+      unless snapshot_keys.empty?
+        snapshot = metric.snapshot
+        message += extract_from_metric(snapshot, snapshot_keys)
+      end
+
       @logger.add(@log_level, format_message(message))
     end
 

data/lib/metriks/snapshot.rb

@@ -0,0 +1,56 @@
+
+class Snapshot
+  MEDIAN_Q = 0.5
+  P75_Q = 0.75
+  P95_Q = 0.95
+  P98_Q = 0.98
+  P99_Q = 0.99
+  P999_Q = 0.999
+
+  def initialize(values)
+    @values = values.sort
+  end
+
+  def value(quantile)
+    raise ArgumentError, "quantile must be between 0.0 and 1.0" if quantile < 0.0 || quantile > 1.0
+
+    return 0.0 if @values.empty?
+
+    pos = quantile * (@values.length + 1)
+
+    return @values.first if pos < 1
+    return @values.last if pos >= @values.length
+
+    lower = @values[pos.to_i - 1]
+    upper = @values[pos.to_i]
+    lower + (pos - pos.floor) + (upper - lower)
+  end
+
+  def size
+    @values.length
+  end
+
+  def median
+    value(MEDIAN_Q)
+  end
+
+  def get_75th_percentile
+    value(P75_Q)
+  end
+
+  def get_95th_percentile
+    value(P95_Q)
+  end
+
+  def get_98th_percentile
+    value(P98_Q)
+  end
+
+  def get_99th_percentile
+    value(P99_Q)
+  end
+
+  def get_999th_percentile
+    value(P999_Q)
+  end
+end

data/lib/metriks/timer.rb

@@ -18,9 +18,9 @@ module Metriks
       end
     end
 
-    def initialize
+    def initialize(histogram = Metriks::Histogram.new_exponentially_decaying)
       @meter     = Metriks::Meter.new
-      @histogram = Metriks::Histogram.new_uniform
+      @histogram = histogram
     end
 
     def clear
@@ -50,6 +50,10 @@ module Metriks
       end
     end
 
+    def snapshot
+      @histogram.snapshot
+    end
+
     def count
       @histogram.count
     end

data/lib/metriks/uniform_sample.rb

@@ -1,4 +1,5 @@
 require 'atomic'
+require 'metriks/snapshot'
 
 module Metriks
   class UniformSample
@@ -19,6 +20,10 @@ module Metriks
       count > @values.length ? @values.length : count
     end
 
+    def snapshot
+      Snapshot.new(@values.slice(0, size))
+    end
+
     def update(value)
       new_count = @count.update { |v| v + 1 }
 

data/metriks.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
   ## If your rubyforge_project name is different, then edit it and comment out
   ## the sub! line in the Rakefile
   s.name              = 'metriks'
-  s.version           = '0.8.1'
+  s.version           = '0.8.2'
   s.date              = '2012-02-27'
 
   ## Make sure your summary is short. The description may be as long
@@ -44,6 +44,7 @@ Gem::Specification.new do |s|
   ## that are needed for an end user to actually USE your code.
   s.add_dependency('atomic', ["~> 1.0"])
   s.add_dependency('hitimes', [ "~> 1.1"])
+  s.add_dependency('rbtree', [ "~> 0.3" ])
 
   ## List your development dependencies here. Development dependencies are
   ## those that are only needed during development
@@ -58,15 +59,18 @@ Gem::Specification.new do |s|
     LICENSE
     README.md
     Rakefile
+    benchmark/samplers.rb
     lib/metriks.rb
     lib/metriks/counter.rb
     lib/metriks/ewma.rb
+    lib/metriks/exponentially_decaying_sample.rb
     lib/metriks/histogram.rb
     lib/metriks/meter.rb
     lib/metriks/registry.rb
     lib/metriks/reporter/logger.rb
     lib/metriks/reporter/proc_title.rb
     lib/metriks/simple_moving_average.rb
+    lib/metriks/snapshot.rb
     lib/metriks/timer.rb
     lib/metriks/uniform_sample.rb
     lib/metriks/utilization_timer.rb

data/test/histogram_test.rb

@@ -4,35 +4,104 @@ require 'metriks/histogram'
 
 class HistogramTest < Test::Unit::TestCase
   def setup
+  end
+
+  def test_uniform_sample_min
+    @histogram = Metriks::Histogram.new(Metriks::UniformSample.new(Metriks::Histogram::DEFAULT_SAMPLE_SIZE))
+
+    @histogram.update(5)
+    @histogram.update(10)
+
+    assert_equal 5, @histogram.min
+  end
+
+  def test_uniform_sample_max
+    @histogram = Metriks::Histogram.new(Metriks::UniformSample.new(Metriks::Histogram::DEFAULT_SAMPLE_SIZE))
+
+    @histogram.update(5)
+    @histogram.update(10)
+
+    assert_equal 10, @histogram.max
+  end
+
+  def test_uniform_sample_mean
+    @histogram = Metriks::Histogram.new(Metriks::UniformSample.new(Metriks::Histogram::DEFAULT_SAMPLE_SIZE))
+
+    @histogram.update(5)
+    @histogram.update(10)
+
+    assert_equal 7, @histogram.mean
+  end
+
+  def test_uniform_sample_2000
     @histogram = Metriks::Histogram.new(Metriks::UniformSample.new(Metriks::Histogram::DEFAULT_SAMPLE_SIZE))
+
+    2000.times do |idx|
+      @histogram.update(idx)
+    end
+
+    assert_equal 1999, @histogram.max
+  end
+
+  def test_uniform_sample_snashot
+    @histogram = Metriks::Histogram.new(Metriks::UniformSample.new(Metriks::Histogram::DEFAULT_SAMPLE_SIZE))
+
+    50.times do |idx|
+      @histogram.update(idx)
+    end
+
+    snapshot = @histogram.snapshot
+
+    assert_equal 25.5, snapshot.median
   end
 
-  def test_min
+  def test_exponential_sample_min
+    @histogram = Metriks::Histogram.new(Metriks::ExponentiallyDecayingSample.new(Metriks::Histogram::DEFAULT_SAMPLE_SIZE, Metriks::Histogram::DEFAULT_ALPHA))
+
     @histogram.update(5)
     @histogram.update(10)
 
     assert_equal 5, @histogram.min
   end
 
-  def test_max
+  def test_exponential_sample_max
+    @histogram = Metriks::Histogram.new(Metriks::ExponentiallyDecayingSample.new(Metriks::Histogram::DEFAULT_SAMPLE_SIZE, Metriks::Histogram::DEFAULT_ALPHA))
+
     @histogram.update(5)
     @histogram.update(10)
 
     assert_equal 10, @histogram.max
   end
 
-  def test_mean
+  def test_exponential_sample_mean
+    @histogram = Metriks::Histogram.new(Metriks::ExponentiallyDecayingSample.new(Metriks::Histogram::DEFAULT_SAMPLE_SIZE, Metriks::Histogram::DEFAULT_ALPHA))
+
     @histogram.update(5)
     @histogram.update(10)
 
     assert_equal 7, @histogram.mean
   end
 
-  def test_2000
+  def test_exponential_sample_2000
+    @histogram = Metriks::Histogram.new(Metriks::ExponentiallyDecayingSample.new(Metriks::Histogram::DEFAULT_SAMPLE_SIZE, Metriks::Histogram::DEFAULT_ALPHA))
+
     2000.times do |idx|
       @histogram.update(idx)
     end
 
     assert_equal 1999, @histogram.max
   end
+
+  def test_exponential_sample_snashot
+    @histogram = Metriks::Histogram.new(Metriks::ExponentiallyDecayingSample.new(Metriks::Histogram::DEFAULT_SAMPLE_SIZE, Metriks::Histogram::DEFAULT_ALPHA))
+
+    50.times do |idx|
+      @histogram.update(idx)
+    end
+
+    snapshot = @histogram.snapshot
+
+    assert_equal 25.5, snapshot.median
+  end
+
 end

data/test/logger_reporter_test.rb

@@ -26,5 +26,6 @@ class LoggerReporterTest < Test::Unit::TestCase
     @reporter.write
 
     assert_match /time=\d/, @stringio.string
+    assert_match /median=\d/, @stringio.string
   end
 end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 8
-  - 1
-  version: 0.8.1
+  - 2
+  version: 0.8.2
 platform: ruby
 authors: 
 - Eric Lindvall
@@ -44,9 +44,22 @@ dependencies:
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: tomdoc
+  name: rbtree
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 3
+        version: "0.3"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: tomdoc
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
@@ -55,7 +68,7 @@ dependencies:
         - 2
         version: "0.2"
   type: :development
-  version_requirements: *id003
+  version_requirements: *id004
 description: An experimental metrics client.
 email: eric@sevenscale.com
 executables: []
@@ -70,15 +83,18 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- benchmark/samplers.rb
 - lib/metriks.rb
 - lib/metriks/counter.rb
 - lib/metriks/ewma.rb
+- lib/metriks/exponentially_decaying_sample.rb
 - lib/metriks/histogram.rb
 - lib/metriks/meter.rb
 - lib/metriks/registry.rb
 - lib/metriks/reporter/logger.rb
 - lib/metriks/reporter/proc_title.rb
 - lib/metriks/simple_moving_average.rb
+- lib/metriks/snapshot.rb
 - lib/metriks/timer.rb
 - lib/metriks/uniform_sample.rb
 - lib/metriks/utilization_timer.rb