samoli-hirefire 0.1.1

data/lib/hirefire/backend/delayed_job/active_record.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+
+module HireFire
+  module Backend
+    module DelayedJob
+      module ActiveRecord
+
+        ##
+        # Counts the amount of queued jobs in the database,
+        # failed jobs are excluded from the sum
+        #
+        # @return [Fixnum] the amount of pending jobs
+        def jobs
+          ::Delayed::Job.
+          where(:failed_at => nil).
+          where('run_at <= ?', Time.now).count
+        end
+
+        ##
+        # Counts the amount of jobs that are locked by a worker
+        #
+        # @return [Fixnum] the amount of (assumably working) workers
+        def workers
+          ::Delayed::Job.
+          where('locked_by IS NOT NULL').count
+        end
+
+      end
+    end
+  end
+end

data/lib/hirefire/backend/delayed_job/mongoid.rb

@@ -0,0 +1,32 @@
+# encoding: utf-8
+
+module HireFire
+  module Backend
+    module DelayedJob
+      module Mongoid
+
+        ##
+        # Counts the amount of queued jobs in the database,
+        # failed jobs and jobs scheduled for the future are excluded
+        #
+        # @return [Fixnum]
+        def jobs
+          ::Delayed::Job.where(
+            :failed_at  => nil,
+            :run_at.lte => Time.now
+          ).count
+        end
+
+        ##
+        # Counts the amount of jobs that are locked by a worker
+        #
+        # @return [Fixnum] the amount of (assumably working) workers
+        def workers
+          ::Delayed::Job.
+          where(:locked_by.ne => nil).count
+        end
+
+      end
+    end
+  end
+end

data/lib/hirefire/backend/resque/redis.rb

@@ -0,0 +1,20 @@
+# encoding: utf-8
+
+module HireFire
+  module Backend
+    module Resque
+      module Redis
+
+        ##
+        # Counts the amount of queued jobs in the database,
+        # failed jobs and jobs scheduled for the future are excluded
+        #
+        # @return [Fixnum]
+        def jobs
+          ::Resque.info[:pending].to_i + ::Resque.info[:working].to_i
+        end
+
+      end
+    end
+  end
+end

data/lib/hirefire/configuration.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+
+module HireFire
+  class Configuration
+
+    ##
+    # Contains the max amount of workers that are allowed to run concurrently
+    #
+    # @return [Fixnum] default: 1
+    attr_accessor :max_workers
+    
+    ##
+    # Contains the min amount of workers that should always be running
+    #
+    # @return [Fixnum] default: 0
+    attr_accessor :min_workers
+
+    ##
+    # Contains the job/worker ratio which determines
+    # how many workers need to be running depending on
+    # the amount of pending jobs
+    #
+    # @return [Array] containing one or more hashes
+    attr_accessor :job_worker_ratio
+
+    ##
+    # Default is nil, in which case it'll auto-detect either :heroku or :noop,
+    # depending on the environment. It will never use :local, unless explicitly defined by the user.
+    #
+    # @param [Symbol, nil] environment Contains the name of the environment to run in.
+    # @return [Symbol, nil] default: nil
+    attr_accessor :environment
+
+    ##
+    # Instantiates a new HireFire::Configuration object
+    # with the default configuration. These default configurations
+    # may be overwritten using the HireFire.configure class method
+    #
+    # @return [HireFire::Configuration]
+    def initialize
+      @max_workers      = 1
+      @min_workers      = 0
+      @job_worker_ratio = [
+          { :jobs => 1,   :workers => 1 },
+          { :jobs => 25,  :workers => 2 },
+          { :jobs => 50,  :workers => 3 },
+          { :jobs => 75,  :workers => 4 },
+          { :jobs => 100, :workers => 5 }
+        ]
+    end
+
+  end
+end

data/lib/hirefire/environment.rb

@@ -0,0 +1,103 @@
+# encoding: utf-8
+
+module HireFire
+  module Environment
+
+    ##
+    # This module gets included in either:
+    #  - Delayed::Backend::ActiveRecord::Job
+    #  - Delayed::Backend::Mongoid::Job
+    #  - Resque::Job
+    #
+    # One of these classes will then be provided with an instance of one of the following:
+    #  - HireFire::Environment::Heroku
+    #  - HireFire::Environment::Local
+    #  - HireFire::Environment::Noop
+    #
+    # This instance is stored in the Class.environment class method
+    #
+    # The Delayed Job classes receive 3 hooks:
+    #  - hirefire_hire      ( invoked when a job gets queued )
+    #  - environment.fire   ( invoked when a queued job gets destroyed )
+    #  - environment.fire   ( invoked when a queued job gets updated unless the job didn't fail )
+    #
+    # The Resque classes get their hooks injected from the HireFire::Initializer#initialize! method
+    #
+    # @param (Class) base This is the class in which this module will be included
+    def self.included(base)
+      base.send :extend, HireFire::Environment::ClassMethods
+
+      ##
+      # Only implement these hooks for Delayed::Job backends
+      if base.name =~ /Delayed::Backend::(ActiveRecord|Mongoid)::Job/
+        base.send :extend, HireFire::Environment::DelayedJob::ClassMethods
+
+        base.class_eval do
+          after_create  'self.class.hirefire_hire'
+          after_destroy 'self.class.environment.fire'
+          after_update  'self.class.environment.fire',
+            :unless => Proc.new { |job| job.failed_at.nil? }
+        end
+      end
+
+      Logger.message("#{ base.name } detected!")
+    end
+
+    ##
+    # Class methods that will be added to the
+    # Delayed::Job and Resque::Job classes
+    module ClassMethods
+
+      ##
+      # Returns the environment class method (containing an instance of the proper environment class)
+      # for either Delayed::Job or Resque::Job
+      #
+      # If HireFire.configuration.environment is nil (the default) then it'll
+      # auto-detect which environment to run in (either Heroku or Noop)
+      #
+      # If HireFire.configuration.environment isn't nil (explicitly set) then
+      # it'll run in the specified environment (Heroku, Local or Noop)
+      #
+      # @return [HireFire::Environment::Heroku, HireFire::Environment::Local, HireFire::Environment::Noop]
+      def environment
+        @environment ||= HireFire::Environment.const_get(
+          if environment = HireFire.configuration.environment
+            environment.to_s.camelize
+          else
+            ENV.include?('HEROKU_UPID') ? 'Heroku' : 'Noop'
+          end
+        ).new
+      end
+    end
+
+    ##
+    # Delayed Job specific module
+    module DelayedJob
+      module ClassMethods
+
+        ##
+        # This method is an attempt to improve web-request throughput.
+        #
+        # A class method for Delayed::Job which first checks if any worker is currently
+        # running by checking to see if there are any jobs locked by a worker. If there aren't
+        # any jobs locked by a worker there is a high chance that there aren't any workers running.
+        # If this is the case, then we sure also invoke the 'self.class.environment.hire' method
+        #
+        # Another check is to see if there is only 1 job (which is the one that
+        # was just added before this callback invoked). If this is the case
+        # then it's very likely there aren't any workers running and we should
+        # invoke the 'self.class.environment.hire' method to make sure this is the case.
+        #
+        # @return [nil]
+        def hirefire_hire
+          delayed_job = ::Delayed::Job.new
+          if delayed_job.workers == 0 \
+          or delayed_job.jobs    == 1
+            environment.hire
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/hirefire/environment/base.rb

@@ -0,0 +1,213 @@
+# encoding: utf-8
+
+module HireFire
+  module Environment
+    class Base
+
+      ##
+      # Include HireFire::Backend helpers
+      include HireFire::Backend
+
+      ##
+      # This method gets invoked when a new job has been queued
+      #
+      # Iterates through the default (or user-defined) job/worker ratio until
+      # it finds a match for the for the current situation (see example).
+      #
+      # @example
+      #   # Say we have 40 queued jobs, and we configured our job/worker ratio like so:
+      #
+      #   HireFire.configure do |config|
+      #     config.max_workers      = 5
+      #     config.job_worker_ratio = [
+      #       { :jobs => 1,   :workers => 1 },
+      #       { :jobs => 15,  :workers => 2 },
+      #       { :jobs => 35,  :workers => 3 },
+      #       { :jobs => 60,  :workers => 4 },
+      #       { :jobs => 80,  :workers => 5 }
+      #     ]
+      #   end
+      #
+      #   # It'll match at { :jobs => 35, :workers => 3 }, (35 jobs or more: hire 3 workers)
+      #   # meaning that it'll ensure there are 3 workers running.
+      #
+      #   # If there were already were 3 workers, it'll leave it as is.
+      #
+      #   # Alternatively, you can use a more functional syntax, which works in the same way.
+      #
+      #   HireFire.configure do |config|
+      #     config.max_workers = 5
+      #     config.job_worker_ratio = [
+      #       { :when => lambda {|jobs| jobs < 15 }, :workers => 1 },
+      #       { :when => lambda {|jobs| jobs < 35 }, :workers => 2 },
+      #       { :when => lambda {|jobs| jobs < 60 }, :workers => 3 },
+      #       { :when => lambda {|jobs| jobs < 80 }, :workers => 4 }
+      #     ]
+      #   end
+      #
+      #   # If there were more than 3 workers running (say, 4 or 5), it will NOT reduce
+      #   # the number. This is because when you reduce the number of workers, you cannot
+      #   # tell which worker Heroku will shut down, meaning you might interrupt a worker
+      #   # that's currently working, causing the job to fail. Also, consider the fact that
+      #   # there are, for example, 35 jobs still to be picked up, so the more workers,
+      #   # the faster it processes. You aren't even paying more because it doesn't matter whether
+      #   # you have 1 worker, or 5 workers processing jobs, because workers are pro-rated to the second.
+      #   # So basically 5 workers would cost 5 times more, but will also process 5 times faster.
+      #
+      #   # Once all jobs finished processing (e.g. Delayed::Job.jobs == 0), HireFire will invoke a signal
+      #   # which will set the workers back to 0 and shuts down all the workers simultaneously.
+      #
+      # @return [nil]
+      def hire
+        jobs_count    = jobs
+        workers_count = workers
+
+        ##
+        # Use "Standard Notation"
+        if not ratio.first[:when].respond_to? :call
+
+          ##
+          # Since the "Standard Notation" is defined in the in an ascending order
+          # in the array of hashes, we need to reverse this order in order to properly
+          # loop through and break out of the array at the correctly matched ratio
+          ratio.reverse!
+
+          ##
+          # Iterates through all the defined job/worker ratio's
+          # until it finds a match. Then it hires (if necessary) the appropriate
+          # amount of workers and breaks out of the loop
+          ratio.each do |ratio|
+
+            ##
+            # Standard notation
+            # This is the code for the default notation
+            #
+            # @example
+            #   { :jobs => 35,  :workers => 3 }
+            #
+            if jobs_count >= ratio[:jobs] and max_workers >= ratio[:workers]
+              if workers_count < ratio[:workers]
+                log_and_hire(ratio[:workers])
+              end
+
+              return
+            end
+          end
+
+          ##
+          # If no match is found in the above job/worker ratio loop, then we'll
+          # perform one last operation to see whether the the job count is greater
+          # than the highest job/worker ratio, and if this is the case then we also
+          # check to see whether the maximum amount of allowed workers is greater
+          # than the amount that are currently running, if this is the case, we are
+          # allowed to hire the max amount of workers.
+          if jobs_count >= ratio.first[:jobs] and max_workers > workers_count
+            log_and_hire(max_workers)
+            return
+          end
+        end
+
+        ##
+        # Use "Functional (Lambda) Notation"
+        if ratio.first[:when].respond_to? :call
+
+          ##
+          # Iterates through all the defined job/worker ratio's
+          # until it finds a match. Then it hires (if necessary) the appropriate
+          # amount of workers and breaks out of the loop
+          ratio.each do |ratio|
+
+            ##
+            # Functional (Lambda) Notation
+            # This is the code for the Lambda notation, more verbose,
+            # but more humanly understandable
+            #
+            # @example
+            #   { :when => lambda {|jobs| jobs < 60 }, :workers => 3 }
+            #
+            if ratio[:when].call(jobs_count) and max_workers >= ratio[:workers]
+              if workers_count < ratio[:workers]
+                log_and_hire(ratio[:workers])
+              end
+
+              break
+            end
+          end
+        end
+
+        ##
+        # Applies only to the Functional (Lambda) Notation
+        # If the amount of queued jobs exceeds that of which was defined in the
+        # job/worker ratio array, it will hire the maximum amount of workers
+        #
+        # "if statements":
+        #   1. true if we use the Functional (Lambda) Notation
+        #   2. true if the last ratio (highest job/worker ratio) was exceeded
+        #   3. true if the max amount of workers are not yet running
+        #
+        # If all the the above statements are true, HireFire will hire the maximum
+        # amount of workers that were specified in the configuration
+        #
+        if ratio.last[:when].respond_to? :call \
+        and ratio.last[:when].call(jobs_count) === false \
+        and max_workers != workers_count
+          log_and_hire(max_workers)
+        end
+      end
+
+      ##
+      # This method gets invoked when a job is either "destroyed"
+      # or "updated, unless the job didn't fail"
+      #
+      # If there are workers active, but there are no more pending jobs,
+      # then fire all the workers or set to the minimum_workers
+      #
+      # @return [nil]
+      def fire
+        if jobs == 0 and workers > min_workers
+          Logger.message("All queued jobs have been processed. " + (min_workers > 0 ? "Setting workers to #{min_workers}." : "Firing all workers."))
+          workers(min_workers)
+        end
+      end
+
+      private
+
+      ##
+      # Helper method for hire that logs the hiring of more workers, then hires those workers.
+      #
+      # @return [nil]
+      def log_and_hire(amount)
+        Logger.message("Hiring more workers so we have #{ amount } in total.")
+        workers(amount)
+      end
+
+      ##
+      # Wrapper method for HireFire.configuration
+      # Returns the max amount of workers that may run concurrently
+      #
+      # @return [Fixnum] the max amount of workers that are allowed to run concurrently
+      def max_workers
+        HireFire.configuration.max_workers
+      end
+      
+      ##
+      # Wrapper method for HireFire.configuration
+      # Returns the min amount of workers that should always be running
+      #
+      # @return [Fixnum] the min amount of workers that should always be running
+      def min_workers
+        HireFire.configuration.min_workers
+      end
+
+      ##
+      # Wrapper method for HireFire.configuration
+      # Returns the job/worker ratio array (in reversed order)
+      #
+      # @return [Array] the array of hashes containing the job/worker ratio
+      def ratio
+        HireFire.configuration.job_worker_ratio
+      end
+
+    end
+  end
+end