job_boss 0.6.8 → 0.7.5

data/README.markdown

@@ -92,6 +92,14 @@ You can even define a block to provide updates on progress (the value which is p
         puts "We're now at #{progress * 100}%"
     end
 
+Prioritization of jobs is also supported.  If a particular batch is more important than others, you can specify a higher priority
+
+    batch = Batch.new(:priority => 3)
+
+In practical terms, the priority represents the number of jobs which are pulled from the queue to be processed each cycle, so by wary of increasing your priority beyond your maximum number of employees.  No job queue will suffer from resource starvation, but you can greatly decrease the performance of other queues by over-prioritizing one.
+
+Also note that job_boss uses a prioritized round-robin approach to scheduling jobs, the priority for jobs is increased throughout the run of the job queue, providing an approximation of a first-come first-serve approach to reduce latency.
+
 For performance, it is recommended that you keep your jobs table clean scheduling execution of the `delete_jobs_before` command on the Job model, which will clean all jobs completed before the specified time:
 
     Job.delete_jobs_before(2.days.ago)

data/job_boss.gemspec

@@ -4,7 +4,7 @@ $:.unshift lib unless $:.include?(lib)
 
 Gem::Specification.new do |s|
   s.name        = "job_boss"
-  s.version     = '0.6.8'
+  s.version     = '0.7.5'
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Brian Underwood"]
   s.email       = ["ml+job_boss@semi-sentient.com"]

data/lib/job_boss/batch.rb

@@ -2,15 +2,15 @@ require 'active_support'
 
 module JobBoss
   class Batch
-    attr_accessor :batch_id
+    attr_accessor :batch_id, :priority
 
     extend ActiveSupport::Memoizable
     # Used to queue jobs in a batch
     # Usage:
     #   batch.queue.math.is_prime?(42)
-    def queue
+    def queue(attributes = {})
       require 'job_boss/queuer'
-      Queuer.new(:batch_id => @batch_id)
+      Queuer.new({:priority => @priority, :batch_id => @batch_id}.merge(attributes))
     end
     memoize :queue
 
@@ -23,8 +23,11 @@ module JobBoss
       queue.send(controller).send(action, *args)
     end
 
-    def initialize(batch_id = nil)
-      @batch_id = batch_id || Batch.generate_batch_id
+    def initialize(options = {})
+      options[:priority] ||= 1
+
+      @priority = options[:priority]
+      @batch_id = options[:batch_id] || Batch.generate_batch_id
     end
 
     # Returns ActiveRecord::Relation representing query for jobs in batch

data/lib/job_boss/boss.rb

@@ -18,9 +18,9 @@ module JobBoss
       # Used to queue jobs
       # Usage:
       #   Boss.queue.math.is_prime?(42)
-      def queue
+      def queue(attributes = {})
         require 'job_boss/queuer'
-        Queuer.new
+        Queuer.new(attributes)
       end
       memoize :queue
 
@@ -86,28 +86,45 @@ module JobBoss
       logger.info "Job Boss started"
       logger.info "Employee limit: #{Boss.config.employee_limit}"
 
+      jobs = []
       while true
-        unless (children_count = available_employees) > 0 && Job.pending.count > 0
-          sleep(config.sleep_interval)
-          next
-        end
+        available_employee_count = wait_for_available_employees
 
-        # Go through each pending path / batch so that we don't get stuck just processing
-        # long running jobs which would leave quicker jobs to suffocate
-        Job.pending.select('DISTINCT path, batch_id').reorder(nil).each do |distinct_job|
-          job = Job.pending.order('id').find_by_path_and_batch_id(distinct_job.path, distinct_job.batch_id)
-          next if job.nil?
+        jobs = dequeue_jobs if jobs.empty?
 
+        [available_employee_count, jobs.size].min.times do
+          job = jobs.shift
           job.dispatch(self)
           @running_jobs << job
-
-          children_count -= 1
-          break unless children_count > 0
         end
 
       end
     end
 
+    # Waits until there is at least one available employee and then returns count
+    def wait_for_available_employees
+      until (employee_count = available_employees) > 0 && Job.pending.count > 0
+        sleep(config.sleep_interval)
+      end
+
+      employee_count
+    end
+
+    # Dequeues next set of jobs based on prioritized round robin algorithm
+    # Priority of a particular queue determines how many jobs get pulled from that queue each time we dequeue
+    # A priority adjustment is also done to give greater priority to sets of jobs which have been running longer
+    def dequeue_jobs
+      logger.info "Dequeuing jobs"
+      Job.pending.select('DISTINCT priority, path, batch_id').reorder(nil).collect do |distinct_job|
+        queue_scope = Job.where(:path => distinct_job.path, :batch_id => distinct_job.batch_id)
+
+        # Give queues which have are further along more priority to reduce latency
+        priority_adjustment = ((queue_scope.completed.count.to_f / queue_scope.count) * config.employee_limit).floor
+
+        queue_scope.pending.limit(distinct_job.priority + priority_adjustment)
+      end.flatten.sort_by(&:id)
+    end
+
     def stop
       logger.info "Stopping #{@running_jobs.size} running employees..."
 

data/lib/job_boss/job.rb

@@ -55,7 +55,7 @@ module JobBoss
     end
 
     def batch
-      self.batch_id && Batch.new(self.batch_id)
+      self.batch_id && Batch.new(:batch_id => self.batch_id, :priority => self.priority)
     end
 
     def result
@@ -75,6 +75,7 @@ module JobBoss
       self.result = nil
       self.completed_at = nil
       self.status = nil
+      self.error_class = nil
       self.error_message = nil
       self.error_backtrace = nil
       self.employee_host = nil

data/lib/job_boss/queuer.rb

@@ -1,6 +1,7 @@
 module JobBoss
   class Queuer
     def initialize(attributes = nil)
+      attributes[:priority] ||= 1
       @attributes = attributes || {}
     end
   

data/lib/migrate.rb

@@ -11,6 +11,8 @@ class CreateJobs < ActiveRecord::Migration
       t.datetime :completed_at
       t.string :status
 
+      t.integer :priority, :default => 1, :null => false
+
       t.string :error_class
       t.string :error_message
       t.text :error_backtrace

data/test/unit/job_test.rb

@@ -187,21 +187,78 @@ class DaemonTest < ActiveSupport::TestCase
 
 
     first_jobs = (0..3).collect do
-      Boss.queue.sleep.sleep_for(3)
+      Boss.queue.sleep.sleep_for(4)
     end
-    job2 = Boss.queue.sleep.sleep_for(3)
+    job2 = Boss.queue.sleep.sleep_for(4)
 
-    sleep(1)
+    sleep(2)
 
     assert first_jobs.all? {|job| job.running? }
     assert !job2.running?
-    sleep(3)
+    sleep(4)
     assert first_jobs.all? {|job| !job.running? }
     assert job2.running?
-    sleep(3)
+    sleep(4)
     assert first_jobs.all? {|job| !job.running? }
     assert !job2.running?
 
+
+
+    # Testing queue prioritization
+    first_batch = Batch.new
+    first_jobs = (0..2).collect do
+      first_batch.queue.sleep.sleep_for(4)
+    end
+    second_batch = Batch.new(:priority => 3)
+    second_jobs = (0..2).collect do
+      second_batch.queue.sleep.sleep_for(4)
+    end
+
+    sleep(2)
+
+    assert first_jobs.first.running?
+    assert first_jobs[1..-1].all? {|job| !job.running? }
+    assert second_jobs.all? {|job| job.running? }
+
+    sleep(4)
+
+    assert !first_jobs.first.running?
+    assert first_jobs[1..-1].all? {|job| job.running? }
+    assert second_jobs.all? {|job| !job.running? }
+
+    sleep(4)
+
+
+
+
+    # Testing adjusted priority which prioritizes jobs which are further along
+    first_batch = Batch.new
+    first_jobs = (0..7).collect do
+      first_batch.queue.sleep.sleep_for(4)
+    end
+
+    sleep(4)
+
+    assert first_jobs[0,4].all? {|job| job.running? }
+    assert first_jobs[4,4].all? {|job| !job.running? }
+
+    second_batch = Batch.new
+    second_jobs = (0..3).collect do
+      second_batch.queue.sleep.sleep_for(4)
+    end
+
+    sleep(2)
+
+    assert first_jobs[0,4].all? {|job| !job.running? }
+    assert first_jobs[4,3].all? {|job| job.running? } # First queue gets an extra employee since it's halfway done
+    assert !first_jobs.last.running?
+    assert second_jobs.first.running?
+    assert second_jobs[1,3].all? {|job| !job.running? }
+
+
+
+
+
     stop_daemon
   end
 end

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: job_boss
 version: !ruby/object:Gem::Version 
-  hash: 23
   prerelease: false
   segments: 
   - 0
-  - 6
-  - 8
-  version: 0.6.8
+  - 7
+  - 5
+  version: 0.7.5
 platform: ruby
 authors: 
 - Brian Underwood
@@ -15,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-14 00:00:00 -05:00
+date: 2011-02-27 00:00:00 -05:00
 default_executable: job_boss
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -26,7 +25,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -40,7 +38,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -54,7 +51,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
         segments: 
         - 0
         version: "0"
@@ -172,7 +168,6 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
       segments: 
       - 0
       version: "0"
@@ -181,7 +176,6 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 23
       segments: 
       - 1
       - 3