samoli-hirefire 0.1.1

data/lib/hirefire/workers/resque.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+
+##
+# Load the "HireFire modified"
+# Resque::Job and Resque::Worker classes
+require File.dirname(__FILE__) + '/resque/job'
+require File.dirname(__FILE__) + '/resque/worker'
+
+##
+# HireFire
+# This is a HireFire modified version of
+# the official Resque module
+module ::Resque
+  def self.enqueue(klass, *args)
+    Job.create(queue_from_class(klass), klass, *args)
+
+    ##
+    # HireFire Hook
+    # After a new job gets queued, we command the current environment
+    # to calculate the amount of workers we need to process the jobs
+    # that are currently queued, and hire them accordingly.
+    if ::Resque.info[:working].to_i == 0 \
+    or ::Resque.info[:jobs] == 1
+      ::Resque::Job.environment.hire
+    end
+
+    Plugin.after_enqueue_hooks(klass).each do |hook|
+      klass.send(hook, *args)
+    end
+  end
+end

data/lib/hirefire/workers/resque/job.rb

@@ -0,0 +1,70 @@
+# encoding: utf-8
+
+##
+# HireFire
+# This is a HireFire modified version of
+# the official Resque::Job class
+module ::Resque
+  class Job
+    def perform
+      job = payload_class
+      job_args = args || []
+      job_was_performed = false
+
+      before_hooks  = Plugin.before_hooks(job)
+      around_hooks  = Plugin.around_hooks(job)
+      after_hooks   = Plugin.after_hooks(job)
+      failure_hooks = Plugin.failure_hooks(job)
+
+      begin
+        begin
+          before_hooks.each do |hook|
+            job.send(hook, *job_args)
+          end
+        rescue DontPerform
+          return false
+        end
+
+        if around_hooks.empty?
+          job.perform(*job_args)
+          job_was_performed = true
+        else
+          stack = around_hooks.reverse.inject(nil) do |last_hook, hook|
+            if last_hook
+              lambda do
+                job.send(hook, *job_args) { last_hook.call }
+              end
+            else
+              lambda do
+                job.send(hook, *job_args) do
+                  result = job.perform(*job_args)
+                  job_was_performed = true
+                  result
+                end
+              end
+            end
+          end
+          stack.call
+        end
+
+        ##
+        # HireFire Hook
+        # After a job finishes processing, we invoke the #fire
+        # method on the environment object which will check to see whether
+        # we can fire all the hired workers
+        ::Resque::Job.environment.fire
+
+        after_hooks.each do |hook|
+          job.send(hook, *job_args)
+        end
+
+        return job_was_performed
+
+      rescue Object => e
+        failure_hooks.each { |hook| job.send(hook, e, *job_args) }
+        raise e
+      end
+    end
+
+  end
+end

data/lib/hirefire/workers/resque/tasks.rb

@@ -0,0 +1,21 @@
+# encoding: utf-8
+
+##
+# Load in the official Resque rake tasks
+require 'resque/tasks'
+
+##
+# Overwrite the resque:setup rake task to first load
+# in the application environment before proceeding
+#
+# ENV['QUEUE'] will default to '*' unless it's defined
+# as an environment variable on Heroku or the Local machine
+task 'resque:setup' => :environment do
+  ENV['QUEUE'] ||= '*'
+end
+
+##
+# This is an alias to the "resque:work" task since Heroku doesn't respond
+# to "resque:work", we need to add this alias so Resque can be initialized by Heroku
+desc 'Alias of "resque:work" - This is required for running on Heroku.'
+task 'jobs:work' => 'resque:work'

data/lib/hirefire/workers/resque/worker.rb

@@ -0,0 +1,57 @@
+# encoding: utf-8
+
+##
+# HireFire
+# This is a HireFire modified version of
+# the official Resque::Worker class
+module ::Resque
+  class Worker
+    def work(interval = 5.0, &block)
+      interval = Float(interval)
+      $0 = "resque: Starting"
+      startup
+
+      loop do
+        break if shutdown?
+        ::Resque::Job.environment.hire
+
+        if not @paused and job = reserve
+          log "got: #{job.inspect}"
+          run_hook :before_fork, job
+          working_on job
+
+          if @child = fork
+            rand # Reseeding
+            procline "Forked #{@child} at #{Time.now.to_i}"
+            Process.wait
+          else
+            procline "Processing #{job.queue} since #{Time.now.to_i}"
+            perform(job, &block)
+            exit! unless @cant_fork
+          end
+
+          done_working
+          @child = nil
+        else
+
+          ##
+          # HireFire Hook
+          # After the last job in the queue finishes processing, Resque::Job.jobs will return 0.
+          # This means that there aren't any more jobs to process for any of the workers.
+          # If this is the case it'll command the current environment to fire all the hired workers
+          # and then immediately break out of this infinite loop.
+          if ::Resque::Job.jobs == 0
+            ::Resque::Job.environment.fire
+            break
+          else
+            sleep(interval)
+          end
+
+        end
+      end
+
+    ensure
+      unregister_worker
+    end
+  end
+end

data/spec/configuration_spec.rb

@@ -0,0 +1,70 @@
+# encoding: utf-8
+
+require File.expand_path('../spec_helper', __FILE__)
+
+describe HireFire::Configuration do
+
+  it 'should have defaults' do
+    configuration = HireFire.configuration
+
+    configuration.environment.should      == nil
+    configuration.max_workers.should      == 1
+    configuration.min_workers.should      == 0
+    configuration.job_worker_ratio.should == [
+        { :jobs => 1,   :workers => 1 },
+        { :jobs => 25,  :workers => 2 },
+        { :jobs => 50,  :workers => 3 },
+        { :jobs => 75,  :workers => 4 },
+        { :jobs => 100, :workers => 5 }
+      ]
+  end
+
+  it 'should be configurable' do
+    HireFire.configure do |config|
+      config.environment      = :noop
+      config.max_workers      = 10
+      config.min_workers      = 0
+      config.job_worker_ratio = [
+          { :jobs => 1,   :workers => 1 },
+          { :jobs => 15,  :workers => 2 },
+          { :jobs => 35,  :workers => 3 },
+          { :jobs => 60,  :workers => 4 },
+          { :jobs => 80,  :workers => 5 }
+        ]
+    end
+
+    configuration = HireFire.configuration
+
+    configuration.environment.should      == :noop
+    configuration.max_workers.should      == 10
+    configuration.min_workers.should      == 0
+    configuration.job_worker_ratio.should == [
+        { :jobs => 1,   :workers => 1 },
+        { :jobs => 15,  :workers => 2 },
+        { :jobs => 35,  :workers => 3 },
+        { :jobs => 60,  :workers => 4 },
+        { :jobs => 80,  :workers => 5 }
+      ]
+  end
+
+  it 'should allow functional syntax' do
+    ratio = [
+      { :when => lambda {|jobs| jobs < 15 }, :workers => 1 },
+      { :when => lambda {|jobs| jobs < 35 }, :workers => 2 },
+      { :when => lambda {|jobs| jobs < 60 }, :workers => 3 },
+      { :when => lambda {|jobs| jobs < 80 }, :workers => 4 }
+    ]
+
+    HireFire.configure do |config|
+      config.environment      = :noop
+      config.max_workers      = 10
+      config.job_worker_ratio = ratio
+    end
+
+    configuration = HireFire.configuration
+
+    configuration.environment.should      == :noop
+    configuration.max_workers.should      == 10
+    configuration.job_worker_ratio.should == ratio
+  end
+end

data/spec/environment_spec.rb

@@ -0,0 +1,369 @@
+# encoding: utf-8
+
+require File.expand_path('../spec_helper', __FILE__)
+
+module HireFire
+  module Backend
+    ##
+    # Stub out backend interface inclusion
+    # since it's irrelevant for these tests
+    def self.included(base)
+      base.send(:include, Environment::Stub)
+    end
+  end
+
+  module Environment
+    module Stub
+      ##
+      # Stubbed out since this normally comes from
+      # a sub class like HireFire::Environment::Heroku or
+      # HireFire::Environment::Local
+      def workers(amount = nil)
+        if amount.nil?
+          @_workers ||= 0
+          return @_workers
+        end
+      end
+
+      ##
+      # Allows the specs to stub the workers count
+      # and return the desired amount
+      def workers=(amount)
+        @_workers = amount
+        self.stubs(:workers).with.returns(amount)
+      end
+
+      ##
+      # Returns the amount of jobs
+      # Defaults to: 0
+      def jobs
+        @_jobs ||= 0
+      end
+
+      ##
+      # Allows the specs to stub the queued job count
+      # and return the desired amount
+      def jobs=(amount)
+        @_jobs = amount
+        self.stubs(:jobs).returns(amount)
+      end
+    end
+  end
+end
+
+describe HireFire::Environment::Base do
+
+  let(:base) { HireFire::Environment::Base.new }
+
+  describe 'testing the test setup' do
+    it 'should default the queued job count to 0 for these specs' do
+      base.jobs.should == 0
+    end
+
+    it 'should set the queued job count to 10' do
+      base.jobs = 10
+      base.jobs.should == 10
+    end
+
+    it 'should have zero workers by default' do
+      base.workers.should == 0
+    end
+
+    it 'should set the amount of workers to 10' do
+      base.workers = 10
+      base.workers.should == 10
+    end
+  end
+
+  describe '#fire' do
+    it 'should not fire any workers when there are still jobs in the queue' do
+      base.jobs    = 1
+      base.workers = 1
+
+      base.expects(:workers).with(0).never
+      base.fire
+    end
+
+    it 'should not fire any workers if there arent any workers' do
+      base.jobs    = 1
+      base.workers = 0
+
+      base.expects(:workers).with(0).never
+      base.fire
+    end
+
+    it 'should fire all workers when there arent any jobs' do
+      base.jobs    = 0
+      base.workers = 1
+
+      HireFire::Logger.expects(:message).with('All queued jobs have been processed. Firing all workers.')
+      base.expects(:workers).with(0).once
+      base.fire
+    end
+    
+    it 'should set the workers to minimum workers when there arent any jobs' do
+      base.jobs    = 0
+      base.workers = 10
+      base.stubs(:min_workers).returns(2)
+      
+      HireFire::Logger.expects(:message).with('All queued jobs have been processed. Setting workers to 2.')
+      base.expects(:workers).with(2).once
+      base.fire
+    end
+  end
+
+  describe '#hire' do
+    describe 'the standard notation' do
+      before do
+        base.stubs(:max_workers).returns(5)
+        base.stubs(:ratio).returns([
+          { :jobs => 1,  :workers => 1 },
+          { :jobs => 15, :workers => 2 },
+          { :jobs => 30, :workers => 3 },
+          { :jobs => 60, :workers => 4 },
+          { :jobs => 90, :workers => 5 }
+        ])
+      end
+
+      it 'should request 1 worker' do
+        base.jobs    = 1
+        base.workers = 0
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 1 in total.')
+        base.expects(:workers).with(1).once
+        base.hire
+      end
+
+      it 'should not request 1 worker, since there already is one worker running' do
+        base.jobs    = 5
+        base.workers = 1
+
+        base.expects(:workers).with(1).never
+        base.hire
+      end
+
+      it 'should request 2 workers' do
+        base.jobs    = 15
+        base.workers = 0
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 2 in total.')
+        base.expects(:workers).with(2).once
+        base.hire
+      end
+
+      it 'should request 2 workers' do
+        base.jobs    = 20
+        base.workers = 1
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 2 in total.')
+        base.expects(:workers).with(2).once
+        base.hire
+      end
+
+      it 'should not request 2 workers since we already have 2' do
+        base.jobs    = 25
+        base.workers = 2
+
+        base.expects(:workers).with(2).never
+        base.hire
+      end
+
+      it 'should NEVER lower the worker amount from the #hire method' do
+        base.jobs    = 25 # simulate that 5 jobs are already processed (30 - 5)
+        base.workers = 3  # and 3 workers are hired
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 2 in total.').never
+        base.expects(:workers).with(2).never
+        base.hire
+      end
+
+      it 'should NEVER hire more workers than the #max_workers' do
+        base.jobs    = 100
+        base.workers = 0
+
+        base.stubs(:max_workers).returns(3) # set the max_workers = 3
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 3 in total.').once
+        base.expects(:workers).with(3).once
+        base.hire
+      end
+
+      it 'should not hire 5 workers even if defined in the job/ratio, when the limit is 3, it should hire 3 max' do
+        base.jobs    = 100
+        base.workers = 0
+
+        base.stubs(:max_workers).returns(3) # set the max_workers = 3
+        base.stubs(:ratio).returns([
+          { :jobs => 5, :workers => 5 }
+        ])
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 3 in total.').once
+        base.expects(:workers).with(3).once
+        base.hire
+      end
+
+      it 'should not hire (or invoke) any more workers since the max amount allowed is already running' do
+        base.jobs    = 100
+        base.workers = 3
+
+        base.stubs(:max_workers).returns(3) # set the max_workers = 3
+        base.stubs(:ratio).returns([
+          { :jobs => 5, :workers => 5 }
+        ])
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 3 in total.').never
+        base.expects(:workers).with(3).never
+        base.hire
+      end
+
+      it 'the max_workers option can only "limit" the amount of max_workers when used in the "Standard Notation"' do
+        base.jobs    = 100
+        base.workers = 0
+
+        base.stubs(:max_workers).returns(10) # set the max_workers = 10
+        base.stubs(:ratio).returns([
+          { :jobs => 5, :workers => 5 }
+        ])
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 5 in total.').once
+        base.expects(:workers).with(5).once
+        base.hire
+      end
+
+      it 'should NEVER do API requests to Heroku if the max_workers are already running' do
+        base.jobs    = 100
+        base.workers = 5
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 5 in total.').never
+        base.expects(:workers).with(5).never
+        base.hire
+      end
+    end
+
+    describe 'the Lambda (functional) notation' do
+      before do
+        base.stubs(:max_workers).returns(5)
+        base.stubs(:ratio).returns([
+          { :when => lambda {|jobs| jobs < 15 }, :workers => 1 },
+          { :when => lambda {|jobs| jobs < 30 }, :workers => 2 },
+          { :when => lambda {|jobs| jobs < 60 }, :workers => 3 },
+          { :when => lambda {|jobs| jobs < 90 }, :workers => 4 }
+        ])
+      end
+
+      it 'should request 1 worker' do
+        base.jobs    = 1
+        base.workers = 0
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 1 in total.')
+        base.expects(:workers).with(1).once
+        base.hire
+      end
+
+      it 'should not request 1 worker, since there already is one worker running' do
+        base.jobs    = 5
+        base.workers = 1
+
+        base.expects(:workers).with(1).never
+        base.hire
+      end
+
+      it 'should request 2 workers' do
+        base.jobs    = 15
+        base.workers = 0
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 2 in total.')
+        base.expects(:workers).with(2).once
+        base.hire
+      end
+
+      it 'should request 2 workers' do
+        base.jobs    = 20
+        base.workers = 1
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 2 in total.')
+        base.expects(:workers).with(2).once
+        base.hire
+      end
+
+      it 'should not request 2 workers since we already have 2' do
+        base.jobs    = 25
+        base.workers = 2
+
+        base.expects(:workers).with(2).never
+        base.hire
+      end
+
+      it 'should NEVER lower the worker amount from the #hire method' do
+        base.jobs    = 25 # simulate that 5 jobs are already processed (30 - 5)
+        base.workers = 3  # and 3 workers are hired
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 2 in total.').never
+        base.expects(:workers).with(2).never
+        base.hire
+      end
+
+      it 'should NEVER hire more workers than the #max_workers' do
+        base.jobs    = 100
+        base.workers = 0
+
+        base.stubs(:max_workers).returns(3) # set the max_workers = 3
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 3 in total.').once
+        base.expects(:workers).with(3).once
+        base.hire
+      end
+
+      it 'should not hire 5 workers even if defined in the job/ratio, when the limit is 3, it should hire 3 max' do
+        base.jobs    = 100
+        base.workers = 0
+
+        base.stubs(:max_workers).returns(3) # set the max_workers = 3
+        base.stubs(:ratio).returns([
+          { :when => lambda { |jobs| jobs < 5 }, :workers => 5 }
+        ])
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 3 in total.').once
+        base.expects(:workers).with(3).once
+        base.hire
+      end
+
+      it 'should not hire (or invoke) any more workers since the max amount allowed is already running' do
+        base.jobs    = 100
+        base.workers = 3
+
+        base.stubs(:max_workers).returns(3) # set the max_workers = 3
+        base.stubs(:ratio).returns([
+          { :when => lambda { |jobs| jobs < 5 }, :workers => 5 }
+        ])
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 3 in total.').never
+        base.expects(:workers).with(3).never
+        base.hire
+      end
+
+      it 'the max_workers option can only "limit" the amount of max_workers when used in the "Standard Notation"' do
+        base.jobs    = 100
+        base.workers = 0
+
+        base.stubs(:max_workers).returns(10) # set the max_workers = 10
+        base.stubs(:ratio).returns([
+          { :when => lambda { |jobs| jobs < 5 }, :workers => 5 }
+        ])
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 10 in total.').once
+        base.expects(:workers).with(10).once
+        base.hire
+      end
+
+      it 'should NEVER do API requests to Heroku if the max_workers are already running' do
+        base.jobs    = 100
+        base.workers = 5
+
+        HireFire::Logger.expects(:message).with('Hiring more workers so we have 5 in total.').never
+        base.expects(:workers).with(5).never
+        base.hire
+      end
+    end
+  end
+end