sqskiq 0.0.3 → 0.0.4

data/lib/sqskiq.rb

@@ -7,58 +7,59 @@ require 'sqskiq/batch_process'
 
 module Sqskiq
 
-  def self.bootstrap(options, worker_class)
-    params = [ @aws_access_key_id, @aws_secret_access_key, options[:queue_name] ]
+  ## 
+  # Configures and starts actor system
+  def self.bootstrap(worker_config, worker_class)
+    config = valid_config_from(worker_config)
+    credentials = [ @aws_access_key_id, @aws_secret_access_key, config[:queue_name] ]
     
-    configured_pool_sizes = pool_sizes(options)
+    Celluloid::Actor[:manager]   = @manager   = Manager.new(config[:empty_queue_throttle])
+    Celluloid::Actor[:fetcher]   = @fetcher   = Fetcher.pool(:size => config[:num_fetchers], :args => credentials)
+    Celluloid::Actor[:deleter]   = @deleter   = Deleter.pool(:size => config[:num_deleters], :args => credentials)
+    Celluloid::Actor[:processor] = @processor = Processor.pool(:size => config[:num_workers], :args => worker_class)
+    Celluloid::Actor[:batcher]   = @batcher   = BatchProcessor.pool(:size => config[:num_batches])
     
-    Celluloid::Actor[:manager] = @manager = Manager.new
-    Celluloid::Actor[:fetcher] = @fetcher = Fetcher.pool(:size => configured_pool_sizes[:num_fetchers], :args => params)
-    Celluloid::Actor[:processor] = @processor = Processor.pool(:size => configured_pool_sizes[:num_workers], :args => worker_class)
-    Celluloid::Actor[:batch_processor] = @batch_processor = BatchProcessor.pool(:size => configured_pool_sizes[:num_batches])
-    Celluloid::Actor[:deleter] = @deleter = Deleter.pool(:size => configured_pool_sizes[:num_deleters], :args => params)
-
     configure_signal_listeners
-
+    
     @manager.bootstrap
     while @manager.running? do
       sleep 2
     end
-
-    @fetcher.__shutdown__
-    @batch_processor.__shutdown__
-    @processor.__shutdown__
-    @deleter.__shutdown__
-
     @manager.terminate
   end
   
+  # Subscribes actors to receive system signals
+  # Each actor when receives a signal should execute
+  # appropriate code to exit cleanly
   def self.configure_signal_listeners
     ['SIGTERM', 'TERM', 'SIGINT'].each do |signal|
       trap(signal) do
         @manager.publish('SIGTERM')
-        @batch_processor.publish('SIGTERM')
+        @batcher.publish('SIGTERM')
         @processor.publish('SIGTERM')
       end
     end
   end
   
-  def self.pool_sizes(options)
-    # for now, min processors should be 2
-    num_workers = (options[:processors].nil? || options[:processors].to_i < 2)? 20 : options[:processors]
-    
-    # each fetch brings up to 10 messages to process.
-    # the number of fetchers is a number able to keep all
-    # workers handling messages
-    # TODO: acctualy the min number must be greater than 2 because we are using
-    # celluloid pool, but that will be changed!
+  ##
+  # checks the provided configuration
+  # and add the defaults when not specified
+  def self.valid_config_from(worker_config)
+    num_workers = (worker_config[:processors].nil? || worker_config[:processors].to_i < 2)? 20 : worker_config[:processors]
+    # messy code due to celluloid pool constraint of 2 as min pool size: see spec for better understanding
     num_fetchers = num_workers / 10
     num_fetchers = num_fetchers + 1 if num_workers % 10 > 0
     num_fetchers = 2 if num_fetchers < 2
-    
     num_deleters = num_batches = num_fetchers
-    
-    { num_workers: num_workers, num_fetchers: num_fetchers, num_batches: num_batches, num_deleters: num_deleters }
+        
+    { 
+      num_workers: num_workers, 
+      num_fetchers: num_fetchers, 
+      num_batches: num_batches, 
+      num_deleters: num_deleters,
+      queue_name: worker_config[:queue_name],
+      empty_queue_throttle: worker_config[:empty_queue_throttle] || 0
+    }
   end
   
   def self.configure

data/lib/sqskiq/batch_process.rb

@@ -13,7 +13,7 @@ module Sqskiq
       subscribe_for_shutdown
     end
 
-    def batch_process(messages)
+    def process(messages)
       process_result = []
       messages.each do |message|
         process_result << @processor.future.process(message)
@@ -30,7 +30,7 @@ module Sqskiq
         end
       end
 
-      @manager.async.batch_process_done(success_messages)
+      @manager.async.batch_done(success_messages)
     end
   end
   

data/lib/sqskiq/manager.rb

@@ -7,33 +7,43 @@ module Sqskiq
     include Celluloid
     include Sqskiq::SignalHandler
     
-    def initialize
-      subscribe_for_shutdown
+    @empty_queue = false
+    
+    def initialize(empty_queue_throttle)
+      @empty_queue_throttle = empty_queue_throttle
+      subscribe_for_shutdown      
     end
 
     def bootstrap
       @fetcher = Celluloid::Actor[:fetcher]
-      @batch_processor = Celluloid::Actor[:batch_processor]
+      @batcher = Celluloid::Actor[:batcher]
       @deleter = Celluloid::Actor[:deleter]
 
       new_fetch(@fetcher.size)
     end
 
     def fetch_done(messages)
-      @batch_processor.async.batch_process(messages) unless @shutting_down
+      @empty_queue = messages.empty?
+      @batcher.async.process(messages) unless @shutting_down
     end
 
-    def batch_process_done(messages)
+    def batch_done(messages)
       @deleter.async.delete(messages)
       new_fetch(1)
     end
 
     def new_fetch(num)
-      num.times { @fetcher.async.fetch unless @shutting_down }
+      after(throttle) do
+        num.times { @fetcher.async.fetch unless @shutting_down }
+      end
     end
 
     def running?
-      not (@shutting_down and @deleter.busy_size == 0 and @batch_processor.busy_size == 0)
+      not (@shutting_down and @deleter.busy_size == 0 and @batcher.busy_size == 0)
+    end
+    
+    def throttle
+      @empty_queue ? @empty_queue_throttle : 0
     end
 
   end

data/spec/manager_spec.rb

@@ -1,70 +1,115 @@
 require 'spec_helper'
 
 describe Sqskiq::Manager do
+  subject { Sqskiq::Manager.new(Random.rand(1..100)) }
+  
   let(:deleter) { Object.new }
   let(:fetcher) { Object.new }
-  let(:batch_processor) { Object.new }
+  let(:batcher) { Object.new }
   let(:shutting_down) { false }
-  subject = described_class.new
   
   before do
     subject.instance_variable_set(:@fetcher, fetcher)
     subject.instance_variable_set(:@deleter, deleter)
-    subject.instance_variable_set(:@batch_processor, batch_processor)
+    subject.instance_variable_set(:@batcher, batcher)
     subject.instance_variable_set(:@shutting_down, shutting_down)
   end
 
   describe '#running?' do
-     
     context 'when the actor system is shutting down' do
-     let(:shutting_down) { true }
+      let(:shutting_down) { true }
      
-     describe 'if deleter is not empty' do
-       before { deleter.should_receive(:busy_size).and_return(1) }
-       
-       it 'returns true' do
-         subject.running?.should be_true
-       end
-     end
+      describe 'if deleter is not empty' do
+        before { deleter.should_receive(:busy_size).and_return(1) }
+ 
+        it { should be_running }
+      end
      
-     describe 'if batch_processor is not empty' do
-       before do 
-         deleter.should_receive(:busy_size).and_return(0) 
-         batch_processor.should_receive(:busy_size).and_return(1) 
-       end
+      describe 'if batcher is not empty' do
+        before do 
+          deleter.should_receive(:busy_size).and_return(0) 
+          batcher.should_receive(:busy_size).and_return(1) 
+        end
 
-       it 'returns true' do
-         subject.running?.should be_true
-       end
-     end
+        it { should be_running }
+      end
      
-     describe 'if batch_processor and deleter are empties' do
+      describe 'if batcher and deleter are empties' do
         before do 
           deleter.should_receive(:busy_size).and_return(0) 
-          batch_processor.should_receive(:busy_size).and_return(0) 
+          batcher.should_receive(:busy_size).and_return(0) 
         end
 
-        it 'returns false' do
-          subject.running?.should be_false
-        end
-      end
-       
+        it { should_not be_running }
+      end    
     end
     
     context 'when the actor system is not shutting down' do
       
-      describe 'even if batch_processor and deleter are empties' do
+      describe 'even if batcher and deleter are empties' do
         before do 
           deleter.stub(:busy_size).and_return(0) 
-          batch_processor.stub(:busy_size).and_return(0) 
+          batcher.stub(:busy_size).and_return(0) 
         end
 
-        it 'returns true' do
-          subject.running?.should be_true
-        end
+        it { should be_running }
       end
       
     end
   end
+  
+  describe '#new_fetch' do
+    before do
+      subject.instance_variable_set(:@empty_queue, empty)
+      subject.instance_variable_set(:@empty_queue_throttle, 10)
+    end
+    
+    context 'if queue is not empty' do
+      let(:empty) { false }
+      
+      it 'applies a throttle of 0 seconds' do
+        Sqskiq::Manager.any_instance.should_receive(:after).with(0)
+        subject.new_fetch(1)
+      end
+    end
+    
+    context 'if queue is empty' do
+      let(:empty) { true }
+      
+      it 'applies a throttle of empty_queue_throttle seconds' do
+        Sqskiq::Manager.any_instance.should_receive(:after).with(10)
+        subject.new_fetch(1)
+      end
+    end
+  end
+  
+  describe '#fetch_done' do
+    
+    before { subject.instance_variable_set(:@empty_queue, empty) }
+    
+    context 'when at least one message has been received' do
+      let(:messages) { ['someMessage'] }
+      let(:empty) { true } 
+      
+      it 'sets @empty_queue to false and process the messages' do
+        batcher.should_receive(:async).and_return(batcher)
+        batcher.should_receive(:process).with(messages)
+        subject.fetch_done(messages)
+        subject.instance_variable_get(:@empty_queue).should be_false
+      end
+    end
+    
+    context 'when no messages are received' do
+      let(:messages) { [] }
+      let(:empty) { false } 
 
+      it 'sets @empty_queue to true and does not process the messages' do
+        batcher.should_receive(:async).and_return(batcher)
+        batcher.should_receive(:process).with(messages)
+        subject.fetch_done(messages)
+        subject.instance_variable_get(:@empty_queue).should be_true
+      end
+    end    
+  end
+  
 end

data/spec/sqskiq_spec.rb

@@ -6,11 +6,11 @@ describe Sqskiq do
     let(:options) { [ { processors: 1 }, ].sample }
     
     it 'uses the defaut value of 20' do
-      pool_sizes = Sqskiq.pool_sizes(options)
-      pool_sizes[:num_workers].should eq(20)
-      pool_sizes[:num_fetchers].should eq(2)
-      pool_sizes[:num_batches].should eq(2)
-      pool_sizes[:num_deleters].should eq(2)
+      config = Sqskiq.valid_config_from(options)
+      config[:num_workers].should eq(20)
+      config[:num_fetchers].should eq(2)
+      config[:num_batches].should eq(2)
+      config[:num_deleters].should eq(2)
     end
   end
   
@@ -20,11 +20,11 @@ describe Sqskiq do
       let(:options) { { processors: [ 20, 30, 40 ].sample } }
       
       it 'uses the the given value' do
-        pool_sizes = Sqskiq.pool_sizes(options)
-        pool_sizes[:num_workers].should eq(options[:processors])
-        pool_sizes[:num_fetchers].should eq(options[:processors] / 10)
-        pool_sizes[:num_batches].should eq(options[:processors] / 10)
-        pool_sizes[:num_deleters].should eq(options[:processors] / 10)
+        config = Sqskiq.valid_config_from(options)
+        config[:num_workers].should eq(options[:processors])
+        config[:num_fetchers].should eq(options[:processors] / 10)
+        config[:num_batches].should eq(options[:processors] / 10)
+        config[:num_deleters].should eq(options[:processors] / 10)
       end
       
     end
@@ -33,11 +33,11 @@ describe Sqskiq do
       let(:options) { { processors: [ 21, 31, 41 ].sample } }
       
       it 'uses the the given value for the processors and apply (processors / 10) + 1 for other pool sizes' do
-        pool_sizes = Sqskiq.pool_sizes(options)
-        pool_sizes[:num_workers].should eq(options[:processors])
-        pool_sizes[:num_fetchers].should eq((options[:processors] / 10) + 1)
-        pool_sizes[:num_batches].should eq((options[:processors] / 10) + 1)
-        pool_sizes[:num_deleters].should eq((options[:processors] / 10) + 1)
+        config = Sqskiq.valid_config_from(options)
+        config[:num_workers].should eq(options[:processors])
+        config[:num_fetchers].should eq((options[:processors] / 10) + 1)
+        config[:num_batches].should eq((options[:processors] / 10) + 1)
+        config[:num_deleters].should eq((options[:processors] / 10) + 1)
       end
       
     end
@@ -46,11 +46,11 @@ describe Sqskiq do
       let(:options) { { processors: Random.rand(2..10) } }
       
       it 'uses the the given value for the processors and apply (processors / 10) for other pool sizes' do
-        pool_sizes = Sqskiq.pool_sizes(options)
-        pool_sizes[:num_workers].should eq(options[:processors])
-        pool_sizes[:num_fetchers].should eq((options[:processors] / 10) + 2)
-        pool_sizes[:num_batches].should eq((options[:processors] / 10) + 2)
-        pool_sizes[:num_deleters].should eq((options[:processors] / 10) + 2)
+        config = Sqskiq.valid_config_from(options)
+        config[:num_workers].should eq(options[:processors])
+        config[:num_fetchers].should eq(2)
+        config[:num_batches].should eq(2)
+        config[:num_deleters].should eq(2)
       end
       
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sqskiq
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors: