sqskiq 0.0.1 → 0.0.2

data/lib/sqskiq/batch_process.rb

@@ -1,21 +1,19 @@
 require 'celluloid'
 require 'celluloid/autostart'
+require 'sqskiq/signal_handler'
 
 module Sqskiq
   class BatchProcessor
     include Celluloid
-    include Celluloid::Notifications
-
+    include Sqskiq::SignalHandler
+    
     def initialize
       @manager = Celluloid::Actor[:manager]
       @processor = Celluloid::Actor[:processor]
-
-      subscribe_interrupt
+      subscribe_for_shutdown
     end
 
     def batch_process(messages)
-      p "processing #{messages.size} messages"
-
       process_result = []
       messages.each do |message|
         process_result << @processor.future.process(message)
@@ -23,24 +21,17 @@ module Sqskiq
 
       success_messages = []
       process_result.each do |result|
-        value = result.value
-        if value[:success]
-          success_messages << value[:message]
+
+        unless @shutting_down
+          value = result.value
+          if value[:success]
+            success_messages << value[:message]
+          end
         end
       end
 
       @manager.async.batch_process_done(success_messages)
     end
-
-    def subscribe_interrupt
-      subscribe('SIGINT', :interrupt)
-      subscribe('TERM', :interrupt)
-      subscribe('SIGTERM', :interrupt)
-    end
-
-    def interrupt(signal)
-      self.terminate
-    end
-
   end
+  
 end

data/lib/sqskiq/delete.rb

@@ -5,7 +5,6 @@ require 'sqskiq/aws'
 module Sqskiq
   class Deleter
     include Celluloid
-    include Celluloid::Notifications
     include Sqskiq::AWS
 
     def initialize(aws_access_key_id, aws_secret_access_key, queue_name)

data/lib/sqskiq/fetch.rb

@@ -5,13 +5,10 @@ require 'sqskiq/aws'
 module Sqskiq
   class Fetcher
     include Celluloid
-    include Celluloid::Notifications
     include Sqskiq::AWS
 
     def initialize(aws_access_key_id, aws_secret_access_key, queue_name)
       init_queue(aws_access_key_id, aws_secret_access_key, queue_name)
-      subscribe_interrupt
-
       @manager = Celluloid::Actor[:manager]
     end
 
@@ -20,15 +17,5 @@ module Sqskiq
       @manager.async.fetch_done(messages)
     end
 
-    def subscribe_interrupt
-      subscribe('SIGINT', :interrupt)
-      subscribe('SIGTERM', :interrupt)
-      subscribe('TERM', :interrupt)
-    end
-
-    def interrupt(signal)
-      self.terminate
-    end
-
   end
 end

data/lib/sqskiq/manager.rb

@@ -1,23 +1,22 @@
 require 'celluloid'
 require 'celluloid/autostart'
-
+require 'sqskiq/signal_handler'
 
 module Sqskiq
   class Manager
     include Celluloid
-    include Celluloid::Notifications
-
+    include Sqskiq::SignalHandler
+    
     def initialize
-      @shutting_down = false
-      subscribe_shutting_down
+      subscribe_for_shutdown
     end
 
     def bootstrap
       @fetcher = Celluloid::Actor[:fetcher]
       @batch_processor = Celluloid::Actor[:batch_processor]
       @deleter = Celluloid::Actor[:deleter]
-      #TODO default value. Should be configurable
-      new_fetch(2)
+
+      new_fetch(@fetcher.size)
     end
 
     def fetch_done(messages)
@@ -25,7 +24,7 @@ module Sqskiq
     end
 
     def batch_process_done(messages)
-      @deleter.async.delete(messages) if @deleter.alive?
+      @deleter.async.delete(messages)
       new_fetch(1)
     end
 
@@ -33,18 +32,9 @@ module Sqskiq
       num.times { @fetcher.async.fetch unless @shutting_down }
     end
 
-    def shutting_down(signal)
-      @shutting_down = true
-    end
-
     def running?
-      not (@deleter.busy_size == 0 and @shutting_down)
+      not (@shutting_down and @deleter.busy_size == 0 and @batch_processor.busy_size == 0)
     end
 
-    def subscribe_shutting_down
-      subscribe('SIGINT', :shutting_down)
-      subscribe('TERM', :shutting_down)
-      subscribe('SIGTERM', :shutting_down)
-    end
   end
 end

data/lib/sqskiq/process.rb

@@ -1,36 +1,28 @@
 require 'celluloid'
 require 'celluloid/autostart'
+require 'sqskiq/signal_handler'
 
 module Sqskiq
   class Processor
     include Celluloid
-    include Celluloid::Notifications
+    include Sqskiq::SignalHandler
 
     def initialize(worker_class)
       @worker_instance = worker_class.new
-
-      subscribe_interrupt
+      subscribe_for_shutdown
     end
 
     def process(message)
+      return  { :success => false, :message => message } if @shutting_down
+
       result = true
       begin
         @worker_instance.perform(message)
       rescue Exception => e
         result = false
       end
-      { :success => result, :message => message }
-    end
-
-    def subscribe_interrupt
-      subscribe('SIGINT', :interrupt)
-      subscribe('TERM', :interrupt)
-      subscribe('SIGTERM', :interrupt)
+      return { :success => result, :message => message }
     end
-
-    def interrupt(signal)
-      self.terminate
-    end
-
+    
   end
 end

data/lib/sqskiq/worker.rb

@@ -1,3 +1,5 @@
+require 'active_support/core_ext/class/attribute'
+
 module Sqskiq
   module Worker
 

data/lib/sqskiq.rb

@@ -8,28 +8,17 @@ require 'sqskiq/batch_process'
 module Sqskiq
 
   def self.bootstrap(options, worker_class)
-
     params = [ @aws_access_key_id, @aws_secret_access_key, options[:queue_name] ]
-
+    
+    configured_pool_sizes = pool_sizes(options)
+    
     Celluloid::Actor[:manager] = @manager = Manager.new
-    Celluloid::Actor[:fetcher] = @fetcher = Fetcher.pool(:size => 2, :args => params)
-    Celluloid::Actor[:processor] = @processor = Processor.pool(:size => 20, :args => worker_class)
-    Celluloid::Actor[:batch_processor] = @batch_processor = BatchProcessor.pool(:size => 2)
-    Celluloid::Actor[:deleter] = @deleter = Deleter.pool(:size => 2, :args => params)
-
-    p "pid = #{Process.pid}"
-
-    trap('SIGTERM') do
-      @manager.publish('SIGTERM')
-    end
-
-    trap('TERM') do
-      @manager.publish('TERM')
-    end
+    Celluloid::Actor[:fetcher] = @fetcher = Fetcher.pool(:size => configured_pool_sizes[:num_fetchers], :args => params)
+    Celluloid::Actor[:processor] = @processor = Processor.pool(:size => configured_pool_sizes[:num_workers], :args => worker_class)
+    Celluloid::Actor[:batch_processor] = @batch_processor = BatchProcessor.pool(:size => configured_pool_sizes[:num_batches])
+    Celluloid::Actor[:deleter] = @deleter = Deleter.pool(:size => configured_pool_sizes[:num_deleters], :args => params)
 
-    trap('SIGINT') do
-      @manager.publish('SIGINT')
-    end
+    configure_signal_listeners
 
     @manager.bootstrap
     while @manager.running? do
@@ -43,7 +32,35 @@ module Sqskiq
 
     @manager.terminate
   end
-
+  
+  def self.configure_signal_listeners
+    ['SIGTERM', 'TERM', 'SIGINT'].each do |signal|
+      trap(signal) do
+        @manager.publish('SIGTERM')
+        @batch_processor.publish('SIGTERM')
+        @processor.publish('SIGTERM')
+      end
+    end
+  end
+  
+  def self.pool_sizes(options)
+    # for now, min processors should be 2
+    num_workers = (options[:processors].nil? || options[:processors].to_i < 2)? 20 : options[:processors]
+    
+    # each fetch brings up to 10 messages to process.
+    # the number of fetchers is a number able to keep all
+    # workers handling messages
+    # TODO: acctualy the min number must be greater than 2 because we are using
+    # celluloid pool, but that will be changed!
+    num_fetchers = num_workers / 10
+    num_fetchers = num_fetchers + 1 if num_workers % 10 > 0
+    num_fetchers = 2 if num_fetchers < 2
+    
+    num_deleters = num_batches = num_fetchers
+    
+    { num_workers: num_workers, num_fetchers: num_fetchers, num_batches: num_batches, num_deleters: num_deleters }
+  end
+  
   def self.configure
     yield self
   end

data/spec/manager_spec.rb

@@ -0,0 +1,70 @@
+require 'spec_helper'
+
+describe Sqskiq::Manager do
+  let(:deleter) { Object.new }
+  let(:fetcher) { Object.new }
+  let(:batch_processor) { Object.new }
+  let(:shutting_down) { false }
+  subject = described_class.new
+  
+  before do
+    subject.instance_variable_set(:@fetcher, fetcher)
+    subject.instance_variable_set(:@deleter, deleter)
+    subject.instance_variable_set(:@batch_processor, batch_processor)
+    subject.instance_variable_set(:@shutting_down, shutting_down)
+  end
+
+  describe '#running?' do
+     
+    context 'when the actor system is shutting down' do
+     let(:shutting_down) { true }
+     
+     describe 'if deleter is not empty' do
+       before { deleter.should_receive(:busy_size).and_return(1) }
+       
+       it 'returns true' do
+         subject.running?.should be_true
+       end
+     end
+     
+     describe 'if batch_processor is not empty' do
+       before do 
+         deleter.should_receive(:busy_size).and_return(0) 
+         batch_processor.should_receive(:busy_size).and_return(1) 
+       end
+
+       it 'returns true' do
+         subject.running?.should be_true
+       end
+     end
+     
+     describe 'if batch_processor and deleter are empties' do
+        before do 
+          deleter.should_receive(:busy_size).and_return(0) 
+          batch_processor.should_receive(:busy_size).and_return(0) 
+        end
+
+        it 'returns false' do
+          subject.running?.should be_false
+        end
+      end
+       
+    end
+    
+    context 'when the actor system is not shutting down' do
+      
+      describe 'even if batch_processor and deleter are empties' do
+        before do 
+          deleter.stub(:busy_size).and_return(0) 
+          batch_processor.stub(:busy_size).and_return(0) 
+        end
+
+        it 'returns true' do
+          subject.running?.should be_true
+        end
+      end
+      
+    end
+  end
+
+end

data/spec/processor_spec.rb

@@ -0,0 +1,14 @@
+require 'spec_helper'
+
+describe Sqskiq::Processor do
+  let(:worker) { double }
+  subject { described_class.new double(new: worker) }
+
+  let(:message) { double(body: 'hello!') }
+
+  context "Error handling" do
+    before { worker.stub(:perform) { raise Exception.new "OMG!" } }
+
+    it { expect(subject.process(message)[:success]).to be_false }
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require 'sqskiq'

data/spec/sqskiq_spec.rb

@@ -0,0 +1,60 @@
+require 'spec_helper'
+
+describe Sqskiq do
+
+  describe 'number of processors is lesser than 2' do
+    let(:options) { [ { processors: 1 }, ].sample }
+    
+    it 'uses the defaut value of 20' do
+      pool_sizes = Sqskiq.pool_sizes(options)
+      pool_sizes[:num_workers].should eq(20)
+      pool_sizes[:num_fetchers].should eq(2)
+      pool_sizes[:num_batches].should eq(2)
+      pool_sizes[:num_deleters].should eq(2)
+    end
+  end
+  
+  describe 'number of processor is greater than 2' do
+    
+    describe 'with nothing remaining after performing division by 10' do
+      let(:options) { { processors: [ 20, 30, 40 ].sample } }
+      
+      it 'uses the the given value' do
+        pool_sizes = Sqskiq.pool_sizes(options)
+        pool_sizes[:num_workers].should eq(options[:processors])
+        pool_sizes[:num_fetchers].should eq(options[:processors] / 10)
+        pool_sizes[:num_batches].should eq(options[:processors] / 10)
+        pool_sizes[:num_deleters].should eq(options[:processors] / 10)
+      end
+      
+    end
+    
+    describe 'with remaining value after performing division by 10' do
+      let(:options) { { processors: [ 21, 31, 41 ].sample } }
+      
+      it 'uses the the given value for the processors and apply (processors / 10) + 1 for other pool sizes' do
+        pool_sizes = Sqskiq.pool_sizes(options)
+        pool_sizes[:num_workers].should eq(options[:processors])
+        pool_sizes[:num_fetchers].should eq((options[:processors] / 10) + 1)
+        pool_sizes[:num_batches].should eq((options[:processors] / 10) + 1)
+        pool_sizes[:num_deleters].should eq((options[:processors] / 10) + 1)
+      end
+      
+    end
+    
+    describe 'and lesser or equals to 10' do
+      let(:options) { { processors: Random.rand(2..10) } }
+      
+      it 'uses the the given value for the processors and apply (processors / 10) for other pool sizes' do
+        pool_sizes = Sqskiq.pool_sizes(options)
+        pool_sizes[:num_workers].should eq(options[:processors])
+        pool_sizes[:num_fetchers].should eq((options[:processors] / 10) + 2)
+        pool_sizes[:num_batches].should eq((options[:processors] / 10) + 2)
+        pool_sizes[:num_deleters].should eq((options[:processors] / 10) + 2)
+      end
+      
+    end
+  end
+  
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sqskiq
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -43,6 +43,38 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.9.1
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: sqskiq
 email: ri.vanlazar@gmail.com
 executables: []
@@ -57,6 +89,10 @@ files:
 - lib/sqskiq/worker.rb
 - lib/sqskiq/batch_process.rb
 - lib/sqskiq/aws.rb
+- spec/manager_spec.rb
+- spec/processor_spec.rb
+- spec/spec_helper.rb
+- spec/sqskiq_spec.rb
 homepage: http://rubygems.org/gems/sqskiq
 licenses: []
 post_install_message: 
@@ -81,4 +117,8 @@ rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: sqskiq
-test_files: []
+test_files:
+- spec/manager_spec.rb
+- spec/processor_spec.rb
+- spec/spec_helper.rb
+- spec/sqskiq_spec.rb