pulse-meter 0.3.1 → 0.3.2

data/lib/pulse-meter.rb

@@ -8,6 +8,9 @@ require "pulse-meter/observer"
 require "pulse-meter/sensor"
 require "pulse-meter/sensor/configuration"
 
+require "pulse-meter/command_aggregator/async"
+require "pulse-meter/command_aggregator/sync"
+
 module PulseMeter
   @@redis = nil
 
@@ -18,4 +21,16 @@ module PulseMeter
   def self.redis=(redis)
     @@redis = redis
   end
+
+  def self.command_aggregator
+    @@command_aggregator ||= PulseMeter::CommandAggregator::Async.instance
+  end
+
+  def self.command_aggregator=(command_aggregator)
+    @@command_aggregator = case command_aggregator
+      when :sync; PulseMeter::CommandAggregator::Sync.instance
+      when :async; PulseMeter::CommandAggregator::Async.instance
+      else raise ArgumentError
+    end
+  end
 end

data/lib/pulse-meter/command_aggregator/async.rb

@@ -0,0 +1,82 @@
+require 'singleton'
+require 'thread'
+
+module PulseMeter
+  module CommandAggregator
+    class Async
+      include Singleton
+
+      MAX_QUEUE_LENGTH = 10_000
+
+      attr_reader :max_queue_length
+
+      def initialize
+        @max_queue_length = MAX_QUEUE_LENGTH
+        @queue = Queue.new
+        @buffer = []
+        @in_multi = false
+        @consumer_thread = run_consumer
+      end
+
+      def multi
+        @in_multi = true
+        yield
+      ensure
+        @in_multi = false
+        send_buffer_to_queue
+      end
+
+      def method_missing(*args)
+        @buffer << args
+        send_buffer_to_queue unless @in_multi
+      end
+
+      def wait_for_pending_events(max_seconds = 1)
+        left_to_wait = max_seconds.to_f
+        sleep_step = 0.01
+        while has_pending_events? && left_to_wait > 0
+          left_to_wait -= sleep_step
+          sleep(sleep_step)
+        end
+      end
+
+      private
+
+      def has_pending_events?
+        !@queue.empty?
+      end
+
+      def send_buffer_to_queue
+        if @queue.size < @max_queue_length
+          @queue << @buffer
+        end
+        @buffer = []
+      end
+
+      def redis
+        PulseMeter.redis
+      end
+
+      def consume_commands
+        # redis and @queue are threadsafe
+        while commands = @queue.pop
+          begin
+            redis.multi do 
+              commands.each do |command|
+                redis.send(*command)
+              end
+            end
+          rescue StandardError => e
+            STDERR.puts "error in consumer_thread: #{e}, #{e.backtrace.join("\n")}"
+          end
+        end
+      end
+
+      def run_consumer
+        Thread.new do
+          consume_commands
+        end
+      end
+    end
+  end
+end

data/lib/pulse-meter/command_aggregator/sync.rb

@@ -0,0 +1,18 @@
+require 'singleton'
+
+module PulseMeter
+  module CommandAggregator
+    class Sync
+      include Singleton
+
+      def redis
+        PulseMeter.redis
+      end
+
+      def method_missing(*args, &block)
+        redis.send(*args, &block)
+      end
+    end
+  end
+end
+

data/lib/pulse-meter/sensor/base.rb

@@ -31,6 +31,10 @@ module PulseMeter
         PulseMeter.redis
       end
 
+      def command_aggregator
+        PulseMeter.command_aggregator
+      end
+
       # Saves annotation to Redis
       # @param description [String] Sensor annotation
       def annotate(description)
@@ -54,7 +58,7 @@ module PulseMeter
       def event(value)
         process_event(value)
         true
-      rescue StandardError
+      rescue StandardError => e
         false
       end
 

data/lib/pulse-meter/sensor/counter.rb

@@ -31,7 +31,7 @@ module PulseMeter
       # Processes event by incremnting counter by given value
       # @param value [Fixnum] increment
       def process_event(value)
-        redis.incrby(value_key, value.to_i)
+        command_aggregator.incrby(value_key, value.to_i)
       end
 
     end

data/lib/pulse-meter/sensor/hashed_counter.rb

@@ -26,8 +26,8 @@ module PulseMeter
       #   and values are increments for their keys
       def process_event(data)
         data.each_pair do |k, v|
-          redis.hincrby(value_key, k, v.to_i)
-          redis.hincrby(value_key, :total, v.to_i)
+          command_aggregator.hincrby(value_key, k, v.to_i)
+          command_aggregator.hincrby(value_key, :total, v.to_i)
         end
       end
 

data/lib/pulse-meter/sensor/hashed_indicator.rb

@@ -16,7 +16,7 @@ module PulseMeter
       # Sets indicator values
       # @param value [Hash] new indicator values
       def process_event(events)
-        events.each_pair {|name, value| redis.hset(value_key, name, value.to_f)}
+        events.each_pair {|name, value| command_aggregator.hset(value_key, name, value.to_f)}
       end
 
     end

data/lib/pulse-meter/sensor/indicator.rb

@@ -27,7 +27,7 @@ module PulseMeter
       # Sets indicator value
       # @param value [Float] new indicator value
       def process_event(value)
-        redis.set(value_key, value.to_f)
+        command_aggregator.set(value_key, value.to_f)
       end
 
     end

data/lib/pulse-meter/sensor/timeline.rb

@@ -57,8 +57,11 @@ module PulseMeter
           interval_id = get_interval_id(time)
           key = raw_data_key(interval_id)
           aggregate_event(key, value)
-          redis.expire(key, raw_data_ttl)
+          command_aggregator.expire(key, raw_data_ttl)
         end
+        true
+      rescue StandardError => e
+        false
       end
 
       # Reduces data in given interval. 
@@ -247,10 +250,10 @@ module PulseMeter
       # Processes event
       # @param value event value
       def process_event(value = nil)
-        multi do
+        command_aggregator.multi do
           current_key = current_raw_data_key
           aggregate_event(current_key, value)
-          redis.expire(current_key, raw_data_ttl)
+          command_aggregator.expire(current_key, raw_data_ttl)
         end
       end
 

data/lib/pulse-meter/sensor/timelined/average.rb

@@ -5,8 +5,8 @@ module PulseMeter
       class Average < Timeline
 
         def aggregate_event(key, value)
-          redis.hincrby(key, :count, 1)
-          redis.hincrby(key, :sum, value)
+          command_aggregator.hincrby(key, :count, 1)
+          command_aggregator.hincrby(key, :sum, value)
         end
 
         def summarize(key)

data/lib/pulse-meter/sensor/timelined/counter.rb

@@ -4,7 +4,7 @@ module PulseMeter
       # Counts events per interval
       class Counter < Timeline
         def aggregate_event(key, value)
-          redis.incrby(key, value.to_i)
+          command_aggregator.incrby(key, value.to_i)
         end
 
         def summarize(key)

data/lib/pulse-meter/sensor/timelined/hashed_counter.rb

@@ -8,8 +8,8 @@ module PulseMeter
       class HashedCounter < Timeline
         def aggregate_event(key, data)
           data.each_pair do |k, v|
-            redis.hincrby(key, k, v)
-            redis.hincrby(key, :total, v)
+            command_aggregator.hincrby(key, k, v)
+            command_aggregator.hincrby(key, :total, v)
           end
         end
 

data/lib/pulse-meter/sensor/timelined/hashed_indicator.rb

@@ -8,7 +8,7 @@ module PulseMeter
       class HashedIndicator < Timeline
         def aggregate_event(key, data)
           data.each_pair do |k, v|
-            redis.hset(key, k, v) if v.respond_to?(:to_f)
+            command_aggregator.hset(key, k, v) if v.respond_to?(:to_f)
           end
         end
 

data/lib/pulse-meter/sensor/timelined/indicator.rb

@@ -4,7 +4,7 @@ module PulseMeter
       # Saves last registered flag float value for each interval
       class Indicator < Timeline
         def aggregate_event(key, value)
-          redis.set(key, value.to_f)
+          command_aggregator.set(key, value.to_f)
         end
 
         def summarize(key)

data/lib/pulse-meter/sensor/timelined/max.rb

@@ -5,8 +5,8 @@ module PulseMeter
       class Max < Timeline
 
         def aggregate_event(key, value)
-          redis.zadd(key, value, "#{value}::#{uniqid}")
-          redis.zremrangebyrank(key, 0, -2)
+          command_aggregator.zadd(key, value, "#{value}::#{uniqid}")
+          command_aggregator.zremrangebyrank(key, 0, -2)
         end
 
         def summarize(key)

data/lib/pulse-meter/sensor/timelined/min.rb

@@ -5,8 +5,8 @@ module PulseMeter
       class Min < Timeline
 
         def aggregate_event(key, value)
-          redis.zadd(key, value, "#{value}::#{uniqid}")
-          redis.zremrangebyrank(key, 1, -1)
+          command_aggregator.zadd(key, value, "#{value}::#{uniqid}")
+          command_aggregator.zremrangebyrank(key, 1, -1)
         end
 
         def summarize(key)

data/lib/pulse-meter/sensor/timelined/multi_percentile.rb

@@ -12,7 +12,7 @@ module PulseMeter
         end
 
         def aggregate_event(key, value)
-          redis.zadd(key, value, "#{value}::#{uniqid}")
+          command_aggregator.zadd(key, value, "#{value}::#{uniqid}")
         end
 
         def summarize(key)

data/lib/pulse-meter/sensor/timelined/percentile.rb

@@ -11,7 +11,7 @@ module PulseMeter
         end
 
         def aggregate_event(key, value)
-          redis.zadd(key, value, "#{value}::#{uniqid}")
+          command_aggregator.zadd(key, value, "#{value}::#{uniqid}")
         end
 
         def summarize(key)

data/lib/pulse-meter/sensor/timelined/uniq_counter.rb

@@ -4,7 +4,7 @@ module PulseMeter
       # Counts unique events per interval
       class UniqCounter < Timeline
         def aggregate_event(key, value)
-          redis.sadd(key, value)
+          command_aggregator.sadd(key, value)
         end
 
         def summarize(key)

data/lib/pulse-meter/sensor/uniq_counter.rb

@@ -16,7 +16,7 @@ module PulseMeter
       # Processes event
       # @param name [String] value to be counted
       def process_event(name)
-        redis.sadd(value_key, name)
+        command_aggregator.sadd(value_key, name)
       end
 
     end

data/lib/pulse-meter/version.rb

@@ -1,3 +1,3 @@
 module PulseMeter
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
 end

data/pulse-meter.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency('terminal-table')
   gem.add_runtime_dependency('thor')
 
+  gem.add_development_dependency('aquarium')
   gem.add_development_dependency('coffee-script')
   gem.add_development_dependency('foreman')
   gem.add_development_dependency('hashie')

data/spec/pulse_meter/command_aggregator/async_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+describe PulseMeter::CommandAggregator::Async do
+  let(:ca){PulseMeter.command_aggregator}
+  let(:redis){PulseMeter.redis}
+
+  describe "#multi" do
+    it "should accumulate redis command and execute in a bulk" do
+      ca.multi do
+        ca.set("xxxx", "zzzz")
+        ca.set("yyyy", "zzzz")
+        sleep 0.1
+        redis.get("xxxx").should be_nil
+        redis.get("yyyy").should be_nil
+      end
+      ca.wait_for_pending_events
+      redis.get("xxxx").should == "zzzz"
+      redis.get("yyyy").should == "zzzz"
+    end
+  end
+
+  describe "any other redis instance method" do
+    it "should be delegated to redis" do
+      ca.set("xxxx", "zzzz")
+      ca.wait_for_pending_events
+      redis.get("xxxx").should == "zzzz"
+    end
+
+    it "should be aggregated if queue is not overflooded" do
+      redis.set("x", 0)
+      ca.max_queue_length.times{ ca.incr("x") }
+      ca.wait_for_pending_events
+      redis.get("x").to_i.should == ca.max_queue_length
+    end
+
+    it "should not be aggregated if queue is overflooded" do
+      redis.set("x", 0)
+      (ca.max_queue_length * 2).times{ ca.incr("x") }
+      ca.wait_for_pending_events
+      redis.get("x").to_i.should < 2 * ca.max_queue_length
+    end
+  end
+
+  describe "#wait_for_pending_events" do
+    it "should pause execution until aggregator thread sends all commands ro redis" do
+      ca.set("xxxx", "zzzz")
+      redis.get("xxxx").should be_nil
+      ca.wait_for_pending_events
+      redis.get("xxxx").should == "zzzz"
+    end
+  end
+
+end

data/spec/pulse_meter/command_aggregator/sync_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+describe PulseMeter::CommandAggregator::Sync do
+  let(:ca){described_class.instance}
+  let(:redis){PulseMeter.redis}
+
+  describe "#multi" do
+    it "should accumulate redis command and execute in a bulk" do
+      ca.multi do
+        ca.set("xxxx", "zzzz").should == "QUEUED"
+        ca.set("yyyy", "zzzz").should == "QUEUED"
+      end
+      redis.get("xxxx").should == "zzzz"
+      redis.get("yyyy").should == "zzzz"
+    end
+  end
+
+  describe "any other redis instance method" do
+    it "should be delegated to redis" do
+      ca.set("xxxx", "zzzz")
+      redis.get("xxxx").should == "zzzz"
+    end
+  end
+end
+

data/spec/pulse_meter_spec.rb

@@ -13,4 +13,41 @@ describe PulseMeter do
       PulseMeter.redis.should == 'redis'
     end
   end
+  describe "::command_aggregator=" do
+    context "when :async passed" do
+      it "should set async command_aggregator to be used" do
+        PulseMeter.command_aggregator = :async
+        PulseMeter.command_aggregator.should be_kind_of(PulseMeter::CommandAggregator::Async)
+      end
+    end
+    context "when :sync passed" do
+      it "should set sync command_aggregator to be used" do
+        PulseMeter.command_aggregator = :sync
+        PulseMeter.command_aggregator.should be_kind_of(PulseMeter::CommandAggregator::Sync)
+      end
+    end
+    context "otherwise" do
+      it "should raise ArgumentError" do
+        expect{ PulseMeter.command_aggregator = :xxx }.to raise_exception(ArgumentError)
+      end
+    end
+  end
+
+  describe "::command_aggregator" do
+    it "should return current command_aggregator" do
+      PulseMeter.command_aggregator = :async
+      PulseMeter.command_aggregator.should be_kind_of(PulseMeter::CommandAggregator::Async)
+      PulseMeter.command_aggregator = :sync
+      PulseMeter.command_aggregator.should be_kind_of(PulseMeter::CommandAggregator::Sync)
+    end
+
+    it "should always return the same command_aggregator for each type" do
+      PulseMeter.command_aggregator = :async
+      ca1 = PulseMeter.command_aggregator
+      PulseMeter.command_aggregator = :sync
+      PulseMeter.command_aggregator = :async
+      ca2 = PulseMeter.command_aggregator
+      ca1.should == ca2
+    end
+  end
 end

data/spec/shared_examples/timeline_sensor.rb

@@ -59,7 +59,7 @@ shared_examples_for "timeline sensor" do |extra_init_values, default_event|
   describe "#event" do
     it "should write events to redis" do
       expect{
-          sensor.event(sample_event)
+        sensor.event(sample_event)
       }.to change{ redis.keys('*').count }.by(1)
     end
 
@@ -100,7 +100,7 @@ shared_examples_for "timeline sensor" do |extra_init_values, default_event|
     it "should write data so that it totally expires after :raw_data_ttl" do
       key_count = redis.keys('*').count
       sensor.event_at(now, sample_event)
-      Timecop.freeze(Time.now + raw_data_ttl + 1) do
+      Timecop.freeze(now + raw_data_ttl + 1) do
         redis.keys('*').count.should == key_count
       end
     end

data/spec/spec_helper.rb

@@ -15,6 +15,16 @@ require 'rack/test'
 Dir['spec/support/**/*.rb'].each{|f| require File.join(ROOT, f) }
 Dir['spec/shared_examples/**/*.rb'].each{|f| require File.join(ROOT,f)}
 
+
+require 'aquarium'
+include Aquarium::Aspects
+  
+Aspect.new :after, :calls_to => [:event, :event_at], :for_types => [PulseMeter::Sensor::Base, PulseMeter::Sensor::Timeline] do |jp, obj, *args|
+  PulseMeter.command_aggregator.wait_for_pending_events
+end
+
+PulseMeter.command_aggregator.max_queue_length = 20
+
 RSpec.configure do |config|
   config.before(:each) do
     PulseMeter.redis = MockRedis.new

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pulse-meter
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-03 00:00:00.000000000 Z
+date: 2012-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: gon-sinatra
@@ -140,6 +140,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: aquarium
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: coffee-script
   requirement: !ruby/object:Gem::Requirement
@@ -397,6 +413,8 @@ files:
 - examples/server_config.yml
 - lib/cmd.rb
 - lib/pulse-meter.rb
+- lib/pulse-meter/command_aggregator/async.rb
+- lib/pulse-meter/command_aggregator/sync.rb
 - lib/pulse-meter/extensions/enumerable.rb
 - lib/pulse-meter/mixins/cmd.rb
 - lib/pulse-meter/mixins/dumper.rb
@@ -515,6 +533,8 @@ files:
 - lib/pulse-meter/visualize/widgets/timeline.rb
 - lib/pulse-meter/visualizer.rb
 - pulse-meter.gemspec
+- spec/pulse_meter/command_aggregator/async_spec.rb
+- spec/pulse_meter/command_aggregator/sync_spec.rb
 - spec/pulse_meter/extensions/enumerable_spec.rb
 - spec/pulse_meter/mixins/cmd_spec.rb
 - spec/pulse_meter/mixins/dumper_spec.rb
@@ -585,6 +605,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 284340155524977771
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23
@@ -593,6 +616,8 @@ specification_version: 3
 summary: Lightweight Redis-based metrics aggregator and processor with CLI and simple
   and customizable WEB interfaces
 test_files:
+- spec/pulse_meter/command_aggregator/async_spec.rb
+- spec/pulse_meter/command_aggregator/sync_spec.rb
 - spec/pulse_meter/extensions/enumerable_spec.rb
 - spec/pulse_meter/mixins/cmd_spec.rb
 - spec/pulse_meter/mixins/dumper_spec.rb