job_boss 0.6.0 → 0.6.5

data/CHANGELOG.markdown

@@ -0,0 +1,10 @@
+## 0.6.5
+
+* Ability to create jobs in batches.  Batches allow for better management of jobs.  Batches for the same task will also run in parallel as opposed to serially
+* Can now call methods on a batch object that one can call as Job class methods.  Examples: wait_for_jobs, result_hash, time_taken, completed_percent
+
+## 0.6.0
+
+* Concept of MIA jobs (boss process marks jobs as MIA when they are killed or otherwise die unhandled)
+* Job.wait_for_jobs method takes a block which allows updating of progress percentage
+* Output in jobs for MIA and cancelled jobs

data/README.markdown

@@ -67,26 +67,28 @@ But since you don't want to do that right now, it looks something like this:
 From your Rails code or in a console:
 
     require 'job_boss'
+    batch = Batch.new
     jobs = (0..1000).collect do |i|
-        Boss.queue.math.is_prime?(i)
+        batch.queue.math.is_prime?(i)
     end
 
 Or:
 
     jobs = []
+    batch = Batch.new
     Article.select('id').find_in_batches(:batch_size => 10) do |articles|
-        jobs << Boss.queue.article.refresh_cache(articles.collect(&:id))
+        jobs << batch.queue.article.refresh_cache(articles.collect(&:id))
     end
 
 job_boss also makes it easy to wait for the jobs to be done and to collect the results into a hash:
 
-    Job.wait_for_jobs(jobs) # Will sleep until the jobs are all complete
+    batch.wait_for_jobs # Will sleep until the jobs are all complete
 
-    Job.result_hash(jobs) # => {[0]=>false, [1]=>false, [2]=>true, [3]=>true, [4]=>false, ... }
+    batch.result_hash # => {[0]=>false, [1]=>false, [2]=>true, [3]=>true, [4]=>false, ... }
 
 You can even define a block to provide updates on progress (the value which is passed into the block is a float between 0.0 and 100.0):
 
-    Job.wait_for_jobs(jobs) do |progress|
+    batch.wait_for_jobs do |progress|
         puts "We're now at #{progress}%"
     end
 

data/job_boss.gemspec

@@ -4,7 +4,7 @@ $:.unshift lib unless $:.include?(lib)
 
 Gem::Specification.new do |s|
   s.name        = "job_boss"
-  s.version     = '0.6.0'
+  s.version     = '0.6.5'
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Brian Underwood"]
   s.email       = ["ml+job_boss@semi-sentient.com"]

data/lib/job_boss.rb

@@ -1,9 +1,11 @@
 require 'job_boss/job'
 require 'job_boss/boss'
+require 'job_boss/batch'
 
 module JobBoss
   autoload :Boss, 'job_boss/boss'
   autoload :Job, 'job_boss/job'
+  autoload :Batch, 'job_boss/batch'
 end
 
 include JobBoss

data/lib/job_boss/batch.rb

@@ -0,0 +1,44 @@
+require 'active_support'
+
+module JobBoss
+  class Batch
+    attr_accessor :batch_id
+
+    extend ActiveSupport::Memoizable
+    # Used to queue jobs in a batch
+    # Usage:
+    #   batch.queue.math.is_prime?(42)
+    def queue
+      require 'job_boss/queuer'
+      Queuer.new(:batch_id => @batch_id)
+    end
+    memoize :queue
+
+    def initialize(batch_id = nil)
+      @batch_id = batch_id || Batch.generate_batch_id
+    end
+
+    # Returns ActiveRecord::Relation representing query for jobs in batch
+    def jobs
+      Job.where('batch_id = ?', @batch_id)
+    end
+
+    # Allow calling of Job class methods from a batch which will be called
+    # on in the scope of the jobs for the batch
+    # Examples: wait_for_jobs, result_hash, time_taken, completed_percent
+    def method_missing(sym, *args, &block)
+      jobs.send(sym, *args, &block)
+    end
+
+  private
+    class << self
+      def generate_batch_id(size = 32)
+        characters = (0..9).to_a + ('a'..'f').to_a
+
+        (1..size).collect do |i|
+          characters[rand(characters.size)]
+        end.join
+      end
+    end
+  end
+end

data/lib/job_boss/boss.rb

@@ -92,10 +92,10 @@ module JobBoss
           next
         end
 
-        # Go through each pending path so that we don't get stuck just processing
+        # Go through each pending path / batch so that we don't get stuck just processing
         # long running jobs which would leave quicker jobs to suffocate
-        Job.pending_paths.each do |path|
-          job = Job.pending.find_by_path(path)
+        Job.pending.select('DISTINCT path, batch_id').each do |distinct_job|
+          job = Job.pending.order('id').find_by_path_and_batch_id(distinct_job.path, distinct_job.batch_id)
           next if job.nil?
 
           job.dispatch(self)

data/lib/job_boss/job.rb

@@ -13,7 +13,7 @@ module JobBoss
     scope :mia, where("completed_at IS NOT NULL AND status = 'mia'")
 
     def prototype
-      self.path + "(#{self.args.join(', ')})"
+      self.path + "(#{self.args.collect(&:inspect).join(', ')})"
     end
 
     # Method used by the boss to dispatch an employee
@@ -54,6 +54,10 @@ module JobBoss
       write_attribute(:result, [value])
     end
 
+    def batch
+      self.batch_id && Batch.new(self.batch_id)
+    end
+
     def result
       # If the result is being called for but the job hasn't been completed, reload
       # to check to see if there was a result
@@ -118,6 +122,15 @@ module JobBoss
       employee_pid && employee_host
     end
 
+    # Is the job running?
+    def running?
+      # If the #running? method is being called for but the job hasn't started, reload
+      # to check to see if it has been assigned
+      self.reload if started_at.nil? || completed_at.nil?
+
+      started_at && !completed_at
+    end
+
     # How long did the job take?
     def time_taken
       # If the #time_taken method is being called for but the job doesn't seem to have started/completed
@@ -127,6 +140,19 @@ module JobBoss
       completed_at - started_at if completed_at && started_at
     end
 
+    # How long did have set of jobs taken?
+    # Returns nil if not all jobs are complete
+    def self.time_taken
+      return nil if self.completed.count != self.count
+
+      self.maximum(:completed_at) - self.minimum(:started_at)
+    end
+
+    # Returns the completion percentage of a set of jobs
+    def self.completed_percent
+      self.completed.count.to_f / self.count.to_f
+    end
+
     # If the job raised an exception, this method will return the instance of that exception
     # with the message and backtrace
     def error
@@ -145,8 +171,9 @@ module JobBoss
       # Given a job or an array of jobs
       # Will cause the process to sleep until all specified jobs have completed
       # sleep_interval specifies polling period
-      def wait_for_jobs(jobs, sleep_interval = 0.5)
+      def wait_for_jobs(jobs = nil, sleep_interval = 0.5)
         jobs = [jobs] if jobs.is_a?(Job)
+        jobs = self.scoped if jobs.nil?
 
         ids = jobs.collect(&:id)
         Job.uncached do
@@ -154,7 +181,7 @@ module JobBoss
             sleep(sleep_interval)
 
             if block_given?
-              yield ((Job.where('id in (?)', ids).completed.count.to_f / jobs.size.to_f) * 100.0)
+              yield((Job.where('id in (?)', ids).completed.count.to_f / jobs.size.to_f) * 100.0)
             end
           end
         end
@@ -165,8 +192,9 @@ module JobBoss
       # Given a job or an array of jobs
       # Returns a hash where the keys are the job method arguments and the values are the
       # results of the job processing
-      def result_hash(jobs)
+      def result_hash(jobs = nil)
         jobs = [jobs] if jobs.is_a?(Job)
+        jobs = self.scoped if jobs.nil?
 
         # the #result method automatically reloads the result here if needed but this will
         # do it in one SQL call
@@ -237,10 +265,6 @@ private
 
         controller_object.send(action, *args)
       end
-
-      def pending_paths
-        self.pending.except(:order).select('DISTINCT path').collect(&:path)
-      end
     end
   end
 end

data/lib/job_boss/queuer.rb

@@ -1,5 +1,9 @@
 module JobBoss
   class Queuer
+    def initialize(attributes = nil)
+      @attributes = attributes || {}
+    end
+  
     def method_missing(method_id, *args)
       require 'active_support'
       require 'job_boss/job'
@@ -17,8 +21,8 @@ module JobBoss
           @class = nil
           @controller = nil
 
-          Job.create(:path => path,
-                     :args => args)
+          Job.create(@attributes.merge(:path => path,
+                                        :args => args))
         else
           raise ArgumentError, "Invalid action"
         end

data/lib/migrate.rb

@@ -2,6 +2,8 @@ class CreateJobs < ActiveRecord::Migration
   def self.up
     create_table :jobs do |t|
       t.string :path
+      t.string :batch_id
+
       t.text :args
       t.text :result
       t.datetime :started_at
@@ -20,6 +22,7 @@ class CreateJobs < ActiveRecord::Migration
     end
 
     add_index :jobs, :path
+    add_index :jobs, :batch_id
     add_index :jobs, :status
 
     postgres = (ActiveRecord::Base.connection.adapter_name == 'PostgreSQL')

data/test/unit/job_test.rb

@@ -138,6 +138,68 @@ class DaemonTest < ActiveSupport::TestCase
     assert_equal 7, job.result
 
 
+    # Test Batch class
+    batch = Batch.new
+
+    jobs = (0..10).collect do |i|
+      batch.queue.math.is_prime?(i)
+    end
+    assert_equal 0.0, batch.completed_percent
+
+    batch.wait_for_jobs
+
+    batch.result_hash.each do |args, result|
+      assert_equal MathJobs.new.is_prime?(args.first), result
+    end
+
+    assert_equal Job.result_hash(jobs), batch.result_hash
+    assert_equal 1.0, batch.completed_percent
+    assert batch.time_taken > 0.5
+
+    # Test to make sure that different batches run in parallel where non-batched jobs don't
+
+    batch1 = Batch.new
+
+    first_jobs = (0..3).collect do
+      batch1.queue.sleep.sleep_for(3)
+    end
+
+    batch2 = Batch.new
+    job2 = batch2.queue.sleep.sleep_for(3)
+
+    sleep(1)
+
+    assert first_jobs[0,3].all? {|job| job.running? }
+    assert !first_jobs.last.running?
+    assert job2.running?
+
+    sleep(3)
+
+    assert first_jobs[0,3].all? {|job| !job.running? }
+    assert first_jobs.last.running?
+    assert !job2.running?
+
+    sleep(3)
+
+    assert !first_jobs.last.running?
+
+
+    first_jobs = (0..3).collect do
+      Boss.queue.sleep.sleep_for(3)
+    end
+    job2 = Boss.queue.sleep.sleep_for(3)
+
+    sleep(1)
+
+    assert first_jobs.all? {|job| job.running? }
+    assert !job2.running?
+    sleep(3)
+    assert first_jobs.all? {|job| !job.running? }
+    assert job2.running?
+    sleep(3)
+    assert first_jobs.all? {|job| !job.running? }
+    assert !job2.running?
+
     stop_daemon
   end
 end

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: job_boss
 version: !ruby/object:Gem::Version 
+  hash: 13
   prerelease: false
   segments: 
   - 0
   - 6
-  - 0
-  version: 0.6.0
+  - 5
+  version: 0.6.5
 platform: ruby
 authors: 
 - Brian Underwood
@@ -14,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-12-22 00:00:00 -05:00
+date: 2011-01-06 00:00:00 -05:00
 default_executable: job_boss
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -25,6 +26,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -38,6 +40,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -51,6 +54,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -66,6 +70,7 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
+- CHANGELOG.markdown
 - Gemfile
 - MIT-LICENSE
 - README.markdown
@@ -134,6 +139,7 @@ files:
 - init.rb
 - job_boss.gemspec
 - lib/job_boss.rb
+- lib/job_boss/batch.rb
 - lib/job_boss/boss.rb
 - lib/job_boss/capistrano.rb
 - lib/job_boss/config.rb
@@ -166,6 +172,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
@@ -174,6 +181,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 23
       segments: 
       - 1
       - 3