hirefire 0.1.0 → 0.1.1

data/lib/hirefire/environment/base.rb

@@ -31,8 +31,20 @@ module HireFire
       #   # It'll match at { :jobs => 35, :workers => 3 }, (35 jobs or more: hire 3 workers)
       #   # meaning that it'll ensure there are 3 workers running.
       #
-      #   # If there were already were 3 workers, it'll leave it as is
-      #   
+      #   # If there were already were 3 workers, it'll leave it as is.
+      #
+      #   # Alternatively, you can use a more functional syntax, which works in the same way.
+      #
+      #   HireFire.configure do |config|
+      #     config.max_workers = 5
+      #     config.job_worker_ratio = [
+      #       { :when => lambda {|jobs| jobs < 15 }, :workers => 1 },
+      #       { :when => lambda {|jobs| jobs < 35 }, :workers => 2 },
+      #       { :when => lambda {|jobs| jobs < 60 }, :workers => 3 },
+      #       { :when => lambda {|jobs| jobs < 80 }, :workers => 4 }
+      #     ]
+      #   end
+      #
       #   # If there were more than 3 workers running (say, 4 or 5), it will NOT reduce
       #   # the number. This is because when you reduce the number of workers, you cannot
       #   # tell which worker Heroku will shut down, meaning you might interrupt a worker
@@ -50,16 +62,97 @@ module HireFire
         jobs_count    = jobs
         workers_count = workers
 
-        ratio.each do |ratio|
-          if jobs_count >= ratio[:jobs] and max_workers >= ratio[:workers]
-            if not workers_count == ratio[:workers]
-              Logger.message("Hiring more workers so we have #{ ratio[:workers] } in total.")
-              workers(ratio[:workers])
+        ##
+        # Use "Standard Notation"
+        if not ratio.first[:when].respond_to? :call
+
+          ##
+          # Since the "Standard Notation" is defined in the in an ascending order
+          # in the array of hashes, we need to reverse this order in order to properly
+          # loop through and break out of the array at the correctly matched ratio
+          ratio.reverse!
+
+          ##
+          # Iterates through all the defined job/worker ratio's
+          # until it finds a match. Then it hires (if necessary) the appropriate
+          # amount of workers and breaks out of the loop
+          ratio.each do |ratio|
+
+            ##
+            # Standard notation
+            # This is the code for the default notation
+            #
+            # @example
+            #   { :jobs => 35,  :workers => 3 }
+            #
+            if jobs_count >= ratio[:jobs] and max_workers >= ratio[:workers]
+              if workers_count < ratio[:workers]
+                log_and_hire(ratio[:workers])
+              end
+
+              return
             end
+          end
+
+          ##
+          # If no match is found in the above job/worker ratio loop, then we'll
+          # perform one last operation to see whether the the job count is greater
+          # than the highest job/worker ratio, and if this is the case then we also
+          # check to see whether the maximum amount of allowed workers is greater
+          # than the amount that are currently running, if this is the case, we are
+          # allowed to hire the max amount of workers.
+          if jobs_count >= ratio.first[:jobs] and max_workers > workers_count
+            log_and_hire(max_workers)
+            return
+          end
+        end
+
+        ##
+        # Use "Functional (Lambda) Notation"
+        if ratio.first[:when].respond_to? :call
+
+          ##
+          # Iterates through all the defined job/worker ratio's
+          # until it finds a match. Then it hires (if necessary) the appropriate
+          # amount of workers and breaks out of the loop
+          ratio.each do |ratio|
 
-            break
+            ##
+            # Functional (Lambda) Notation
+            # This is the code for the Lambda notation, more verbose,
+            # but more humanly understandable
+            #
+            # @example
+            #   { :when => lambda {|jobs| jobs < 60 }, :workers => 3 }
+            #
+            if ratio[:when].call(jobs_count) and max_workers >= ratio[:workers]
+              if workers_count < ratio[:workers]
+                log_and_hire(ratio[:workers])
+              end
+
+              break
+            end
           end
         end
+
+        ##
+        # Applies only to the Functional (Lambda) Notation
+        # If the amount of queued jobs exceeds that of which was defined in the
+        # job/worker ratio array, it will hire the maximum amount of workers
+        #
+        # "if statements":
+        #   1. true if we use the Functional (Lambda) Notation
+        #   2. true if the last ratio (highest job/worker ratio) was exceeded
+        #   3. true if the max amount of workers are not yet running
+        #
+        # If all the the above statements are true, HireFire will hire the maximum
+        # amount of workers that were specified in the configuration
+        #
+        if ratio.last[:when].respond_to? :call \
+        and ratio.last[:when].call(jobs_count) === false \
+        and max_workers != workers_count
+          log_and_hire(max_workers)
+        end
       end
 
       ##
@@ -79,6 +172,15 @@ module HireFire
 
       private
 
+      ##
+      # Helper method for hire that logs the hiring of more workers, then hires those workers.
+      #
+      # @return [nil]
+      def log_and_hire(amount)
+        Logger.message("Hiring more workers so we have #{ amount } in total.")
+        workers(amount)
+      end
+
       ##
       # Wrapper method for HireFire.configuration
       # Returns the max amount of workers that may run concurrently
@@ -92,9 +194,9 @@ module HireFire
       # Wrapper method for HireFire.configuration
       # Returns the job/worker ratio array (in reversed order)
       #
-      # @return [Array] the array of hashes containing the job/worker ratio (in reversed order)
+      # @return [Array] the array of hashes containing the job/worker ratio
       def ratio
-        HireFire.configuration.job_worker_ratio.reverse
+        HireFire.configuration.job_worker_ratio
       end
 
     end

data/lib/hirefire/initializer.rb

@@ -4,33 +4,69 @@ module HireFire
   class Initializer
 
     ##
-    # Loads the HireFire extension in to Delayed Job and
-    # extends the Delayed Job "jobs:work" rake task command
+    # Loads the HireFire extension in to the loaded worker library and
+    # extends that library by injecting HireFire hooks in the proper locations.
+    #
+    # Currently it supports:
+    #  - Delayed Job
+    #    - ActiveRecord ORM
+    #    - Mongoid ODM
+    #  - Resque
+    #    - Redis
+    #
+    # @note
+    #   Either the Delayed Job, or the Resque worker library must be
+    #   loaded BEFORE HireFire initializes, otherwise it'll be unable
+    #   to detect the proper library and it will not work.
     #
     # @return [nil]
     def self.initialize!
+
       ##
-      # If DelayedJob is using ActiveRecord, then include
-      # HireFire::Environment in to the ActiveRecord Delayed Job Backend
-      if defined?(Delayed::Backend::ActiveRecord::Job)
-        Delayed::Backend::ActiveRecord::Job.
-        send(:include, HireFire::Environment).
-        send(:include, HireFire::Backend)
+      # Initialize Delayed::Job extensions if Delayed::Job is found
+      if defined?(::Delayed::Job)
+        ##
+        # If DelayedJob is using ActiveRecord, then include
+        # HireFire::Environment in to the ActiveRecord Delayed Job Backend
+        if defined?(::Delayed::Backend::ActiveRecord::Job)
+          ::Delayed::Backend::ActiveRecord::Job.
+          send(:include, HireFire::Environment).
+          send(:include, HireFire::Backend)
+        end
+
+        ##
+        # If DelayedJob is using Mongoid, then include
+        # HireFire::Environment in to the Mongoid Delayed Job Backend
+        if defined?(::Delayed::Backend::Mongoid::Job)
+          ::Delayed::Backend::Mongoid::Job.
+          send(:include, HireFire::Environment).
+          send(:include, HireFire::Backend)
+        end
+
+        ##
+        # Load Delayed Job extensions, this will patch Delayed::Worker
+        # to implement the necessary hooks to invoke HireFire from
+        require File.join(HireFire::WORKERS_PATH, 'delayed_job')
       end
 
       ##
-      # If DelayedJob is using Mongoid, then include
-      # HireFire::Environment in to the Mongoid Delayed Job Backend
-      if defined?(Delayed::Backend::Mongoid::Job)
-        Delayed::Backend::Mongoid::Job.
+      # Initialize Resque extensions if Resque is found
+      if defined?(::Resque)
+
+        ##
+        # Include the HireFire::Environment which will add an instance
+        # of HireFire::Environment::(Heroku|Local|Noop) to the Resque::Job.environment class method
+        #
+        # Extend the Resque::Job class with the Resque::Job.jobs class method
+        ::Resque::Job.
         send(:include, HireFire::Environment).
-        send(:include, HireFire::Backend)
-      end
+        send(:extend, HireFire::Backend::Resque::Redis)
 
-      ##
-      # Load Delayed Job extension, this is the start
-      # method that gets invoked when running "rake jobs:work"
-      require File.dirname(__FILE__) + '/delayed_job_extension'
+        ##
+        # Load Resque extensions, this will patch Resque, Resque::Job and Resque::Worker
+        # to implement the necessary hooks to invoke HireFire from
+        require File.join(HireFire::WORKERS_PATH, 'resque')
+      end
     end
 
   end

data/lib/hirefire/railtie.rb

@@ -4,11 +4,37 @@ module HireFire
   class Railtie < Rails::Railtie
 
     ##
-    # Initializes HireFire for Delayed Job when
+    # Initializes HireFire for either Delayed Job or Resque when
     # the Ruby on Rails web framework is done loading
+    #
+    # @note
+    #   Either the Delayed Job, or the Resque worker library must be
+    #   loaded BEFORE HireFire initializes, otherwise it'll be unable
+    #   to detect the proper library and it will not work.
     initializer :after_initialize do
       HireFire::Initializer.initialize!
     end
 
+    ##
+    # Adds additional rake tasks to the Ruby on Rails environment
+    #
+    # @note
+    #   In order for Resque to run on Heroku, it must have the 'rake jobs:work'
+    #   rake task since that's what Heroku uses to start workers. When using
+    #   Ruby on Rails automatically add the necessary default rake task for the user
+    #
+    # @note
+    #   Delayed Job already has 'rake jobs:work' built in.
+    #
+    rake_tasks do
+
+      ##
+      # If Resque is loaded, then we load the Resque rake task
+      # that'll allow Heroku to start up Resque as a worker
+      if defined?(::Resque)
+        require File.join(WORKERS_PATH, 'resque', 'tasks.rb')
+      end
+    end
+
   end
 end

data/lib/hirefire/version.rb

@@ -6,7 +6,7 @@ module HireFire
     ##
     # @return [String] the current version of the HireFire gem
     def self.current
-      '0.1.0'
+      '0.1.1'
     end
 
   end

data/lib/hirefire/workers/delayed_job.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+
+##
+# Load the "HireFire modified" Delayed::Worker class
+require File.dirname(__FILE__) + '/delayed_job/worker'

data/lib/hirefire/{delayed_job_extension.rb → workers/delayed_job/worker.rb}

@@ -1,5 +1,9 @@
 # encoding: utf-8
 
+##
+# HireFire
+# This is a HireFire modified version of
+# the official Delayed::Worker class
 module Delayed
   class Worker
 
@@ -42,10 +46,14 @@ module Delayed
         end
 
         ##
-        # If there are no jobs currently queued,
-        # and the worker is still running, it'll kill itself
+        # HireFire Hook
+        # After the last job in the queue finishes processing, Delayed::Job.new.jobs (queued.jobs)
+        # will return 0. This means that there aren't any more jobs to process for any of the workers.
+        # If this is the case it'll command the current environment to fire all the hired workers
+        # and then immediately break out of this infinite loop.
         if queued.jobs == 0
           Delayed::Job.environment.fire
+          break
         end
 
         break if $exit

data/lib/hirefire/workers/resque.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+
+##
+# Load the "HireFire modified"
+# Resque::Job and Resque::Worker classes
+require File.dirname(__FILE__) + '/resque/job'
+require File.dirname(__FILE__) + '/resque/worker'
+
+##
+# HireFire
+# This is a HireFire modified version of
+# the official Resque module
+module ::Resque
+  def self.enqueue(klass, *args)
+    Job.create(queue_from_class(klass), klass, *args)
+
+    ##
+    # HireFire Hook
+    # After a new job gets queued, we command the current environment
+    # to calculate the amount of workers we need to process the jobs
+    # that are currently queued, and hire them accordingly.
+    if ::Resque.info[:working].to_i == 0 \
+    or ::Resque.info[:jobs] == 1
+      ::Resque::Job.environment.hire
+    end
+
+    Plugin.after_enqueue_hooks(klass).each do |hook|
+      klass.send(hook, *args)
+    end
+  end
+end

data/lib/hirefire/workers/resque/job.rb

@@ -0,0 +1,70 @@
+# encoding: utf-8
+
+##
+# HireFire
+# This is a HireFire modified version of
+# the official Resque::Job class
+module ::Resque
+  class Job
+    def perform
+      job = payload_class
+      job_args = args || []
+      job_was_performed = false
+
+      before_hooks  = Plugin.before_hooks(job)
+      around_hooks  = Plugin.around_hooks(job)
+      after_hooks   = Plugin.after_hooks(job)
+      failure_hooks = Plugin.failure_hooks(job)
+
+      begin
+        begin
+          before_hooks.each do |hook|
+            job.send(hook, *job_args)
+          end
+        rescue DontPerform
+          return false
+        end
+
+        if around_hooks.empty?
+          job.perform(*job_args)
+          job_was_performed = true
+        else
+          stack = around_hooks.reverse.inject(nil) do |last_hook, hook|
+            if last_hook
+              lambda do
+                job.send(hook, *job_args) { last_hook.call }
+              end
+            else
+              lambda do
+                job.send(hook, *job_args) do
+                  result = job.perform(*job_args)
+                  job_was_performed = true
+                  result
+                end
+              end
+            end
+          end
+          stack.call
+        end
+
+        ##
+        # HireFire Hook
+        # After a job finishes processing, we invoke the #fire
+        # method on the environment object which will check to see whether
+        # we can fire all the hired workers
+        ::Resque::Job.environment.fire
+
+        after_hooks.each do |hook|
+          job.send(hook, *job_args)
+        end
+
+        return job_was_performed
+
+      rescue Object => e
+        failure_hooks.each { |hook| job.send(hook, e, *job_args) }
+        raise e
+      end
+    end
+
+  end
+end

data/lib/hirefire/workers/resque/tasks.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+
+##
+# Load in the official Resque rake tasks
+require 'resque/tasks'
+
+##
+# Overwrite the resque:setup rake task to first load
+# in the application environment before proceeding
+#
+# ENV['QUEUE'] will default to '*' unless it's defined
+# as an environment variable on Heroku or the Local machine
+task 'resque:setup' => :environment do
+  ENV['QUEUE'] ||= '*'
+end
+
+##
+# This is an alias to the "resque:work" task since Heroku doesn't respond
+# to "resque:work", we need to add this alias so Resque can be initialized by Heroku
+desc 'Alias of "resque:work" - This is required for running on Heroku.'
+task 'jobs:work' => 'resque:work'