hirefire 0.1.0 → 0.1.1

data/lib/hirefire/workers/resque/worker.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+
+##
+# HireFire
+# This is a HireFire modified version of
+# the official Resque::Worker class
+module ::Resque
+  class Worker
+    def work(interval = 5.0, &block)
+      interval = Float(interval)
+      $0 = "resque: Starting"
+      startup
+
+      loop do
+        break if shutdown?
+        ::Resque::Job.environment.hire
+
+        if not @paused and job = reserve
+          log "got: #{job.inspect}"
+          run_hook :before_fork, job
+          working_on job
+
+          if @child = fork
+            rand # Reseeding
+            procline "Forked #{@child} at #{Time.now.to_i}"
+            Process.wait
+          else
+            procline "Processing #{job.queue} since #{Time.now.to_i}"
+            perform(job, &block)
+            exit! unless @cant_fork
+          end
+
+          done_working
+          @child = nil
+        else
+
+          ##
+          # HireFire Hook
+          # After the last job in the queue finishes processing, Resque::Job.jobs will return 0.
+          # This means that there aren't any more jobs to process for any of the workers.
+          # If this is the case it'll command the current environment to fire all the hired workers
+          # and then immediately break out of this infinite loop.
+          if ::Resque::Job.jobs == 0
+            ::Resque::Job.environment.fire
+            break
+          else
+            sleep(interval)
+          end
+
+        end
+      end
+
+    ensure
+      unregister_worker
+    end
+  end
+end

data/spec/configuration_spec.rb

@@ -44,4 +44,24 @@ describe HireFire::Configuration do
       ]
   end
 
+  it 'should allow functional syntax' do
+    ratio = [
+      { :when => lambda {|jobs| jobs < 15 }, :workers => 1 },
+      { :when => lambda {|jobs| jobs < 35 }, :workers => 2 },
+      { :when => lambda {|jobs| jobs < 60 }, :workers => 3 },
+      { :when => lambda {|jobs| jobs < 80 }, :workers => 4 }
+    ]
+
+    HireFire.configure do |config|
+      config.environment      = :noop
+      config.max_workers      = 10
+      config.job_worker_ratio = ratio
+    end
+
+    configuration = HireFire.configuration
+
+    configuration.environment.should      == :noop
+    configuration.max_workers.should      == 10
+    configuration.job_worker_ratio.should == ratio
+  end
 end

data/spec/environment_spec.rb

@@ -0,0 +1,359 @@
+# encoding: utf-8
+
+require File.expand_path('../spec_helper', __FILE__)
+
+module HireFire
+  module Backend
+    ##
+    # Stub out backend interface inclusion
+    # since it's irrelevant for these tests
+    def self.included(base)
+      base.send(:include, Environment::Stub)
+    end
+  end
+
+  module Environment
+    module Stub
+      ##
+      # Stubbed out since this normally comes from
+      # a sub class like HireFire::Environment::Heroku or
+      # HireFire::Environment::Local
+      def workers(amount = nil)
+        if amount.nil?
+          @_workers ||= 0
+          return @_workers
+        end
+      end
+
+      ##
+      # Allows the specs to stub the workers count
+      # and return the desired amount
+      def workers=(amount)
+        @_workers = amount
+        self.stubs(:workers).with.returns(amount)
+      end
+
+      ##
+      # Returns the amount of jobs
+      # Defaults to: 0
+      def jobs
+        @_jobs ||= 0
+      end
+
+      ##
+      # Allows the specs to stub the queued job count
+      # and return the desired amount
+      def jobs=(amount)
+        @_jobs = amount
+        self.stubs(:jobs).returns(amount)
+      end
+    end
+  end
+end
+
+describe HireFire::Environment::Base do
+
+  let(:base) { HireFire::Environment::Base.new }
+
+  describe 'testing the test setup' do
+    it 'should default the queued job count to 0 for these specs' do
+      base.jobs.should == 0
+    end
+
+    it 'should set the queued job count to 10' do
+      base.jobs = 10
+      base.jobs.should == 10
+    end
+
+    it 'should have zero workers by default' do
+      base.workers.should == 0
+    end
+
+    it 'should set the amount of workers to 10' do
+      base.workers = 10
+      base.workers.should == 10
+    end
+  end
+
+  describe '#fire' do
+    it 'should not fire any workers when there are still jobs in the queue' do
+      base.jobs    = 1
+      base.workers = 1
+
+      base.expects(:workers).with(0).never
+      base.fire
+    end
+
+    it 'should not fire any workers if there arent any workers' do
+      base.jobs    = 1
+      base.workers = 0
+
+      base.expects(:workers).with(0).never
+      base.fire
+    end
+
+    it 'should fire all workers when there arent any jobs' do
+      base.jobs    = 0
+      base.workers = 1
+
+      HireFire::Logger.expects(:message).with('All queued jobs have been processed. Firing all workers.')
+      base.expects(:workers).with(0).once
+      base.fire
+    end
+  end
+
+  describe '#hire' do
+    describe 'the standard notation' do
+      before do
+        base.stubs(:max_workers).returns(5)
+        base.stubs(:ratio).returns([
+          { :jobs => 1,  :workers => 1 },
+          { :jobs => 15, :workers => 2 },
+          { :jobs => 30, :workers => 3 },
+          { :jobs => 60, :workers => 4 },
+          { :jobs => 90, :workers => 5 }
+        ])
+      end
+
+      it 'should request 1 worker' do
+        base.jobs    = 1
+        base.workers = 0
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 1 in total.')
+        base.expects(:workers).with(1).once
+        base.hire
+      end
+
+      it 'should not request 1 worker, since there already is one worker running' do
+        base.jobs    = 5
+        base.workers = 1
+
+        base.expects(:workers).with(1).never
+        base.hire
+      end
+
+      it 'should request 2 workers' do
+        base.jobs    = 15
+        base.workers = 0
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 2 in total.')
+        base.expects(:workers).with(2).once
+        base.hire
+      end
+
+      it 'should request 2 workers' do
+        base.jobs    = 20
+        base.workers = 1
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 2 in total.')
+        base.expects(:workers).with(2).once
+        base.hire
+      end
+
+      it 'should not request 2 workers since we already have 2' do
+        base.jobs    = 25
+        base.workers = 2
+
+        base.expects(:workers).with(2).never
+        base.hire
+      end
+
+      it 'should NEVER lower the worker amount from the #hire method' do
+        base.jobs    = 25 # simulate that 5 jobs are already processed (30 - 5)
+        base.workers = 3  # and 3 workers are hired
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 2 in total.').never
+        base.expects(:workers).with(2).never
+        base.hire
+      end
+
+      it 'should NEVER hire more workers than the #max_workers' do
+        base.jobs    = 100
+        base.workers = 0
+
+        base.stubs(:max_workers).returns(3) # set the max_workers = 3
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 3 in total.').once
+        base.expects(:workers).with(3).once
+        base.hire
+      end
+
+      it 'should not hire 5 workers even if defined in the job/ratio, when the limit is 3, it should hire 3 max' do
+        base.jobs    = 100
+        base.workers = 0
+
+        base.stubs(:max_workers).returns(3) # set the max_workers = 3
+        base.stubs(:ratio).returns([
+          { :jobs => 5, :workers => 5 }
+        ])
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 3 in total.').once
+        base.expects(:workers).with(3).once
+        base.hire
+      end
+
+      it 'should not hire (or invoke) any more workers since the max amount allowed is already running' do
+        base.jobs    = 100
+        base.workers = 3
+
+        base.stubs(:max_workers).returns(3) # set the max_workers = 3
+        base.stubs(:ratio).returns([
+          { :jobs => 5, :workers => 5 }
+        ])
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 3 in total.').never
+        base.expects(:workers).with(3).never
+        base.hire
+      end
+
+      it 'the max_workers option can only "limit" the amount of max_workers when used in the "Standard Notation"' do
+        base.jobs    = 100
+        base.workers = 0
+
+        base.stubs(:max_workers).returns(10) # set the max_workers = 10
+        base.stubs(:ratio).returns([
+          { :jobs => 5, :workers => 5 }
+        ])
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 5 in total.').once
+        base.expects(:workers).with(5).once
+        base.hire
+      end
+
+      it 'should NEVER do API requests to Heroku if the max_workers are already running' do
+        base.jobs    = 100
+        base.workers = 5
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 5 in total.').never
+        base.expects(:workers).with(5).never
+        base.hire
+      end
+    end
+
+    describe 'the Lambda (functional) notation' do
+      before do
+        base.stubs(:max_workers).returns(5)
+        base.stubs(:ratio).returns([
+          { :when => lambda {|jobs| jobs < 15 }, :workers => 1 },
+          { :when => lambda {|jobs| jobs < 30 }, :workers => 2 },
+          { :when => lambda {|jobs| jobs < 60 }, :workers => 3 },
+          { :when => lambda {|jobs| jobs < 90 }, :workers => 4 }
+        ])
+      end
+
+      it 'should request 1 worker' do
+        base.jobs    = 1
+        base.workers = 0
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 1 in total.')
+        base.expects(:workers).with(1).once
+        base.hire
+      end
+
+      it 'should not request 1 worker, since there already is one worker running' do
+        base.jobs    = 5
+        base.workers = 1
+
+        base.expects(:workers).with(1).never
+        base.hire
+      end
+
+      it 'should request 2 workers' do
+        base.jobs    = 15
+        base.workers = 0
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 2 in total.')
+        base.expects(:workers).with(2).once
+        base.hire
+      end
+
+      it 'should request 2 workers' do
+        base.jobs    = 20
+        base.workers = 1
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 2 in total.')
+        base.expects(:workers).with(2).once
+        base.hire
+      end
+
+      it 'should not request 2 workers since we already have 2' do
+        base.jobs    = 25
+        base.workers = 2
+
+        base.expects(:workers).with(2).never
+        base.hire
+      end
+
+      it 'should NEVER lower the worker amount from the #hire method' do
+        base.jobs    = 25 # simulate that 5 jobs are already processed (30 - 5)
+        base.workers = 3  # and 3 workers are hired
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 2 in total.').never
+        base.expects(:workers).with(2).never
+        base.hire
+      end
+
+      it 'should NEVER hire more workers than the #max_workers' do
+        base.jobs    = 100
+        base.workers = 0
+
+        base.stubs(:max_workers).returns(3) # set the max_workers = 3
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 3 in total.').once
+        base.expects(:workers).with(3).once
+        base.hire
+      end
+
+      it 'should not hire 5 workers even if defined in the job/ratio, when the limit is 3, it should hire 3 max' do
+        base.jobs    = 100
+        base.workers = 0
+
+        base.stubs(:max_workers).returns(3) # set the max_workers = 3
+        base.stubs(:ratio).returns([
+          { :when => lambda { |jobs| jobs < 5 }, :workers => 5 }
+        ])
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 3 in total.').once
+        base.expects(:workers).with(3).once
+        base.hire
+      end
+
+      it 'should not hire (or invoke) any more workers since the max amount allowed is already running' do
+        base.jobs    = 100
+        base.workers = 3
+
+        base.stubs(:max_workers).returns(3) # set the max_workers = 3
+        base.stubs(:ratio).returns([
+          { :when => lambda { |jobs| jobs < 5 }, :workers => 5 }
+        ])
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 3 in total.').never
+        base.expects(:workers).with(3).never
+        base.hire
+      end
+
+      it 'the max_workers option can only "limit" the amount of max_workers when used in the "Standard Notation"' do
+        base.jobs    = 100
+        base.workers = 0
+
+        base.stubs(:max_workers).returns(10) # set the max_workers = 10
+        base.stubs(:ratio).returns([
+          { :when => lambda { |jobs| jobs < 5 }, :workers => 5 }
+        ])
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 10 in total.').once
+        base.expects(:workers).with(10).once
+        base.hire
+      end
+
+      it 'should NEVER do API requests to Heroku if the max_workers are already running' do
+        base.jobs    = 100
+        base.workers = 5
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 5 in total.').never
+        base.expects(:workers).with(5).never
+        base.hire
+      end
+    end
+  end
+end

metadata

@@ -1,8 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: hirefire
 version: !ruby/object:Gem::Version 
+  hash: 25
   prerelease: 
-  version: 0.1.0
+  segments: 
+  - 0
+  - 1
+  - 1
+  version: 0.1.1
 platform: ruby
 authors: 
 - Michael van Rooijen
@@ -10,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-10 00:00:00 Z
+date: 2011-04-15 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: heroku
@@ -20,6 +25,11 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
+        hash: 69
+        segments: 
+        - 1
+        - 20
+        - 1
         version: 1.20.1
   type: :runtime
   version_requirements: *id001
@@ -31,10 +41,15 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 0
+        - 6
+        - 7
         version: 0.6.7
   type: :runtime
   version_requirements: *id002
-description: "                      HireFire automatically \"hires\" and \"fires\" (aka \"scales\") Delayed Job workers on Heroku.\n                      When there are no queue jobs, HireFire will fire (shut down) all workers. If there are\n                      queued jobs, then it'll hire (spin up) workers. The amount of workers that get hired\n                      depends on the amount of queued jobs (the ratio can be configured by you). HireFire\n                      is great for both high, mid and low traffic applications. It can save you a lot of\n                      money by only hiring workers when there are pending jobs, and then firing them again\n                      once all the jobs have been processed. It's also capable to dramatically reducing\n                      processing time by automatically hiring more workers when the queue size increases.\n"
+description: HireFire automatically "hires" and "fires" (aka "scales") Delayed Job and Resque workers on Heroku. When there are no queue jobs, HireFire will fire (shut down) all workers. If there are queued jobs, then it'll hire (spin up) workers. The amount of workers that get hired depends on the amount of queued jobs (the ratio can be configured by you). HireFire is great for both high, mid and low traffic applications. It can save you a lot of money by only hiring workers when there are pending jobs, and then firing them again once all the jobs have been processed. It's also capable to dramatically reducing processing time by automatically hiring more workers when the queue size increases.
 email: meskyanichi@gmail.com
 executables: []
 
@@ -54,10 +69,10 @@ files:
 - hirefire.gemspec
 - lib/hirefire.rb
 - lib/hirefire/backend.rb
-- lib/hirefire/backend/active_record.rb
-- lib/hirefire/backend/mongoid.rb
+- lib/hirefire/backend/delayed_job/active_record.rb
+- lib/hirefire/backend/delayed_job/mongoid.rb
+- lib/hirefire/backend/resque/redis.rb
 - lib/hirefire/configuration.rb
-- lib/hirefire/delayed_job_extension.rb
 - lib/hirefire/environment.rb
 - lib/hirefire/environment/base.rb
 - lib/hirefire/environment/heroku.rb
@@ -67,7 +82,14 @@ files:
 - lib/hirefire/logger.rb
 - lib/hirefire/railtie.rb
 - lib/hirefire/version.rb
+- lib/hirefire/workers/delayed_job.rb
+- lib/hirefire/workers/delayed_job/worker.rb
+- lib/hirefire/workers/resque.rb
+- lib/hirefire/workers/resque/job.rb
+- lib/hirefire/workers/resque/tasks.rb
+- lib/hirefire/workers/resque/worker.rb
 - spec/configuration_spec.rb
+- spec/environment_spec.rb
 - spec/logger_spec.rb
 - spec/spec_helper.rb
 homepage: http://rubygems.org/gems/hirefire
@@ -83,20 +105,25 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.7.2
+rubygems_version: 1.7.1
 signing_key: 
 specification_version: 3
-summary: HireFire automatically "hires" and "fires" (aka "scales") Delayed Job workers on Heroku.
+summary: HireFire automatically "hires" and "fires" (aka "scales") Delayed Job and Resque workers on Heroku.
 test_files: []
 
-has_rdoc: