hirefire 0.1.1 → 0.1.2

data/README.md

@@ -44,6 +44,7 @@ And that's it. Next time you deploy to [Heroku](http://heroku.com/) it'll automa
     HireFire.configure do |config|
       config.environment      = nil # default in production is :heroku. default in development is :noop
       config.max_workers      = 5   # default is 1
+      config.min_workers      = 0   # default is 0
       config.job_worker_ratio = [
           { :jobs => 1,   :workers => 1 },
           { :jobs => 15,  :workers => 2 },

data/lib/hirefire.rb

@@ -44,8 +44,9 @@ module HireFire
     ##
     # HireFire::Backend::DelayedJob namespace
     module DelayedJob
-      autoload :ActiveRecord, File.join(DELAYED_JOB_PATH, 'active_record')
-      autoload :Mongoid,      File.join(DELAYED_JOB_PATH, 'mongoid')
+      autoload :ActiveRecord,   File.join(DELAYED_JOB_PATH, 'active_record')
+      autoload :ActiveRecord2,  File.join(DELAYED_JOB_PATH, 'active_record_2')
+      autoload :Mongoid,        File.join(DELAYED_JOB_PATH, 'mongoid')
     end
 
     ##
@@ -72,6 +73,7 @@ module HireFire
   #   HireFire.configure do |config|
   #     config.environment      = nil
   #     config.max_workers      = 5
+  #     config.min_workers      = 0
   #     config.job_worker_ratio = [
   #       { :jobs => 1,   :workers => 1 },
   #       { :jobs => 15,  :workers => 2 },
@@ -103,9 +105,10 @@ end
 # in their application manually, after loading the worker library (either "Delayed Job" or "Resque")
 # and the desired mapper (ActiveRecord, Mongoid or Redis)
 if defined?(Rails)
-  if Rails.version >= '3.0.0'
+  if defined?(Rails::Railtie)
     require File.join(HireFire::HIREFIRE_PATH, 'railtie')
   else
     HireFire::Initializer.initialize!
   end
 end
+

data/lib/hirefire/backend.rb

@@ -16,9 +16,13 @@ module HireFire
 
       ##
       # Delayed Job specific backends
-      if defined?(::Delayed::Job)
+      if defined?(::Delayed)
         if defined?(::Delayed::Backend::ActiveRecord::Job)
-          base.send(:include, HireFire::Backend::DelayedJob::ActiveRecord)
+          if defined?(::ActiveRecord::Relation)
+            base.send(:include, HireFire::Backend::DelayedJob::ActiveRecord)
+          else
+            base.send(:include, HireFire::Backend::DelayedJob::ActiveRecord2)
+          end
         end
 
         if defined?(::Delayed::Backend::Mongoid::Job)
@@ -35,3 +39,4 @@ module HireFire
 
   end
 end
+

data/lib/hirefire/backend/delayed_job/active_record.rb

@@ -18,9 +18,11 @@ module HireFire
 
         ##
         # Counts the amount of jobs that are locked by a worker
+        # There is no other performant way to determine the amount
+        # of workers there currently are
         #
         # @return [Fixnum] the amount of (assumably working) workers
-        def workers
+        def working
           ::Delayed::Job.
           where('locked_by IS NOT NULL').count
         end

data/lib/hirefire/backend/delayed_job/active_record_2.rb

@@ -0,0 +1,35 @@
+# encoding: utf-8
+
+module HireFire
+  module Backend
+    module DelayedJob
+      module ActiveRecord2
+
+        ##
+        # Counts the amount of queued jobs in the database,
+        # failed jobs are excluded from the sum
+        #
+        # @return [Fixnum] the amount of pending jobs
+        def jobs
+          ::Delayed::Job.all(
+            :conditions => ['failed_at IS NULL and run_at <= ?', Time.now.utc]
+          ).count
+        end
+
+        ##
+        # Counts the amount of jobs that are locked by a worker
+        # There is no other performant way to determine the amount
+        # of workers there currently are
+        #
+        # @return [Fixnum] the amount of (assumably working) workers
+        def working
+          ::Delayed::Job.all(
+            :conditions => 'locked_by IS NOT NULL'
+          ).count
+        end
+
+      end
+    end
+  end
+end
+

data/lib/hirefire/backend/delayed_job/mongoid.rb

@@ -19,9 +19,11 @@ module HireFire
 
         ##
         # Counts the amount of jobs that are locked by a worker
+        # There is no other performant way to determine the amount
+        # of workers there currently are
         #
         # @return [Fixnum] the amount of (assumably working) workers
-        def workers
+        def working
           ::Delayed::Job.
           where(:locked_by.ne => nil).count
         end

data/lib/hirefire/backend/resque/redis.rb

@@ -6,12 +6,30 @@ module HireFire
       module Redis
 
         ##
-        # Counts the amount of queued jobs in the database,
-        # failed jobs and jobs scheduled for the future are excluded
+        # Counts the amount of pending jobs in Redis
+        #
+        # Failed jobs are excluded because they are not listed as "pending"
+        # and jobs cannot be scheduled for the future in Resque
         #
         # @return [Fixnum]
         def jobs
-          ::Resque.info[:pending].to_i + ::Resque.info[:working].to_i
+          ::Resque.info[:pending].to_i
+        end
+
+        ##
+        # Counts the amount of workers
+        #
+        # @return [Fixnum]
+        def workers
+          ::Resque.info[:workers].to_i
+        end
+
+        ##
+        # Counts the amount of jobs that are being processed by workers
+        #
+        # @return [Fixnum]
+        def working
+          ::Resque.info[:working].to_i
         end
 
       end

data/lib/hirefire/configuration.rb

@@ -9,6 +9,12 @@ module HireFire
     # @return [Fixnum] default: 1
     attr_accessor :max_workers
 
+    ##
+    # Contains the min amount of workers that should always be running
+    #
+    # @return [Fixnum] default: 0
+    attr_accessor :min_workers
+
     ##
     # Contains the job/worker ratio which determines
     # how many workers need to be running depending on
@@ -33,6 +39,7 @@ module HireFire
     # @return [HireFire::Configuration]
     def initialize
       @max_workers      = 1
+      @min_workers      = 0
       @job_worker_ratio = [
           { :jobs => 1,   :workers => 1 },
           { :jobs => 25,  :workers => 2 },

data/lib/hirefire/environment.rb

@@ -91,7 +91,7 @@ module HireFire
         # @return [nil]
         def hirefire_hire
           delayed_job = ::Delayed::Job.new
-          if delayed_job.workers == 0 \
+          if delayed_job.working == 0 \
           or delayed_job.jobs    == 1
             environment.hire
           end

data/lib/hirefire/environment/base.rb

@@ -19,6 +19,7 @@ module HireFire
       #
       #   HireFire.configure do |config|
       #     config.max_workers      = 5
+      #     config.min_workers      = 0
       #     config.job_worker_ratio = [
       #       { :jobs => 1,   :workers => 1 },
       #       { :jobs => 15,  :workers => 2 },
@@ -60,7 +61,7 @@ module HireFire
       # @return [nil]
       def hire
         jobs_count    = jobs
-        workers_count = workers
+        workers_count = workers || return
 
         ##
         # Use "Standard Notation"
@@ -160,13 +161,13 @@ module HireFire
       # or "updated, unless the job didn't fail"
       #
       # If there are workers active, but there are no more pending jobs,
-      # then fire all the workers
+      # then fire all the workers or set to the minimum_workers
       #
       # @return [nil]
       def fire
-        if jobs == 0 and workers > 0
-          Logger.message("All queued jobs have been processed. Firing all workers.")
-          workers(0)
+        if jobs == 0 and workers > min_workers
+          Logger.message("All queued jobs have been processed. " + (min_workers > 0 ? "Setting workers to #{min_workers}." : "Firing all workers."))
+          workers(min_workers)
         end
       end
 
@@ -192,7 +193,16 @@ module HireFire
 
       ##
       # Wrapper method for HireFire.configuration
-      # Returns the job/worker ratio array (in reversed order)
+      # Returns the min amount of workers that should always be running
+      #
+      # @return [Fixnum] the min amount of workers that should always be running
+      def min_workers
+        HireFire.configuration.min_workers
+      end
+
+      ##
+      # Wrapper method for HireFire.configuration
+      # Returns the job/worker ratio array
       #
       # @return [Array] the array of hashes containing the job/worker ratio
       def ratio

data/lib/hirefire/environment/heroku.rb

@@ -31,6 +31,12 @@ module HireFire
         # Sets the amount of Delayed Job
         # workers that need to be running on Heroku
         client.set_workers(ENV['APP_NAME'], amount)
+
+      rescue RestClient::Exception
+        # Heroku library uses rest-client, currently, and it is quite
+        # possible to receive RestClient exceptions through the client.
+        HireFire::Logger.message("Worker query request failed with #{ $!.class.name } #{ $!.message }")
+        nil
       end
 
       ##

data/lib/hirefire/initializer.rb

@@ -24,7 +24,7 @@ module HireFire
 
       ##
       # Initialize Delayed::Job extensions if Delayed::Job is found
-      if defined?(::Delayed::Job)
+      if defined?(::Delayed)
         ##
         # If DelayedJob is using ActiveRecord, then include
         # HireFire::Environment in to the ActiveRecord Delayed Job Backend
@@ -71,3 +71,4 @@ module HireFire
 
   end
 end
+

data/lib/hirefire/version.rb

@@ -6,7 +6,7 @@ module HireFire
     ##
     # @return [String] the current version of the HireFire gem
     def self.current
-      '0.1.1'
+      '0.1.2'
     end
 
   end

data/lib/hirefire/workers/delayed_job/worker.rb

@@ -15,7 +15,13 @@ module Delayed
     #   except for the following:
     #
     #   1. All ouput will now go through the HireFire::Logger.
-    #   2. When HireFire cannot find any jobs to process it sends the "fire"
+    #   2. Invoke the ::Delayed::Job.environment.hire method at every loop
+    #      to see whether we need to hire more workers so that we can delegate
+    #      this task to the workers, rather than the web servers to improve web-throughput
+    #      by avoiding any unnecessary API calls to Heroku.
+    #      If there are any workers running, then the front end will never invoke API calls
+    #      since the worker(s) can handle this itself.
+    #   3. When HireFire cannot find any jobs to process it sends the "fire"
     #      signal to all workers, ending all the processes simultaneously. The reason
     #      we wait for all the processes to finish before sending the signal is because it'll
     #      otherwise interrupt workers and leave jobs unfinished.
@@ -29,6 +35,7 @@ module Delayed
       queued = Delayed::Job.new
 
       loop do
+        ::Delayed::Job.environment.hire
         result = nil
 
         realtime = Benchmark.realtime do

data/lib/hirefire/workers/resque.rb

@@ -16,11 +16,12 @@ module ::Resque
 
     ##
     # HireFire Hook
-    # After a new job gets queued, we command the current environment
-    # to calculate the amount of workers we need to process the jobs
-    # that are currently queued, and hire them accordingly.
-    if ::Resque.info[:working].to_i == 0 \
-    or ::Resque.info[:jobs] == 1
+    # After a new job gets enqueued we check to see if there are currently
+    # any workers up and running. If this is the case then we do nothing and
+    # let the worker pick up the jobs (and potentially hire more workers)
+    #
+    # If there are no workers, then we manually hire workers.
+    if ::Resque::Job.workers == 0
       ::Resque::Job.environment.hire
     end
 

data/lib/hirefire/workers/resque/worker.rb

@@ -40,7 +40,7 @@ module ::Resque
           # This means that there aren't any more jobs to process for any of the workers.
           # If this is the case it'll command the current environment to fire all the hired workers
           # and then immediately break out of this infinite loop.
-          if ::Resque::Job.jobs == 0
+          if (::Resque::Job.jobs + ::Resque::Job.working) == 0
             ::Resque::Job.environment.fire
             break
           else

data/spec/configuration_spec.rb

@@ -9,6 +9,7 @@ describe HireFire::Configuration do
 
     configuration.environment.should      == nil
     configuration.max_workers.should      == 1
+    configuration.min_workers.should      == 0
     configuration.job_worker_ratio.should == [
         { :jobs => 1,   :workers => 1 },
         { :jobs => 25,  :workers => 2 },
@@ -22,6 +23,7 @@ describe HireFire::Configuration do
     HireFire.configure do |config|
       config.environment      = :noop
       config.max_workers      = 10
+      config.min_workers      = 0
       config.job_worker_ratio = [
           { :jobs => 1,   :workers => 1 },
           { :jobs => 15,  :workers => 2 },
@@ -35,6 +37,7 @@ describe HireFire::Configuration do
 
     configuration.environment.should      == :noop
     configuration.max_workers.should      == 10
+    configuration.min_workers.should      == 0
     configuration.job_worker_ratio.should == [
         { :jobs => 1,   :workers => 1 },
         { :jobs => 15,  :workers => 2 },

data/spec/environment_spec.rb

@@ -100,6 +100,16 @@ describe HireFire::Environment::Base do
       base.expects(:workers).with(0).once
       base.fire
     end
+
+    it 'should set the workers to minimum workers when there arent any jobs' do
+      base.jobs    = 0
+      base.workers = 10
+      base.stubs(:min_workers).returns(2)
+
+      HireFire::Logger.expects(:message).with('All queued jobs have been processed. Setting workers to 2.')
+      base.expects(:workers).with(2).once
+      base.fire
+    end
   end
 
   describe '#hire' do
@@ -228,6 +238,15 @@ describe HireFire::Environment::Base do
         base.expects(:workers).with(5).never
         base.hire
       end
+
+      it 'should NEVER do API requests to Heroku if the workers query returns nil' do
+        base.jobs    = 100
+        base.workers = nil
+
+        base.expects(:log_and_hire).never
+        base.expects(:fire).never
+        base.hire
+      end
     end
 
     describe 'the Lambda (functional) notation' do

metadata

@@ -1,13 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: hirefire
 version: !ruby/object:Gem::Version 
-  hash: 25
   prerelease: 
-  segments: 
-  - 0
-  - 1
-  - 1
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors: 
 - Michael van Rooijen
@@ -15,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-15 00:00:00 Z
+date: 2011-05-01 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: heroku
@@ -25,11 +20,6 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 69
-        segments: 
-        - 1
-        - 20
-        - 1
         version: 1.20.1
   type: :runtime
   version_requirements: *id001
@@ -41,11 +31,6 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 9
-        segments: 
-        - 0
-        - 6
-        - 7
         version: 0.6.7
   type: :runtime
   version_requirements: *id002
@@ -70,6 +55,7 @@ files:
 - lib/hirefire.rb
 - lib/hirefire/backend.rb
 - lib/hirefire/backend/delayed_job/active_record.rb
+- lib/hirefire/backend/delayed_job/active_record_2.rb
 - lib/hirefire/backend/delayed_job/mongoid.rb
 - lib/hirefire/backend/resque/redis.rb
 - lib/hirefire/configuration.rb
@@ -105,25 +91,20 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.7.1
+rubygems_version: 1.7.2
 signing_key: 
 specification_version: 3
 summary: HireFire automatically "hires" and "fires" (aka "scales") Delayed Job and Resque workers on Heroku.
 test_files: []
 
+has_rdoc: