sidekiq-superworker 0.0.8 → 0.0.9

data/app/models/sidekiq/superworker/subjob.rb

@@ -2,11 +2,13 @@ module Sidekiq
   module Superworker
     class Subjob < ActiveRecord::Base
       attr_accessible :jid, :subjob_id, :superjob_id, :parent_id, :children_ids, :next_id,
-        :subworker_class, :superworker_class, :arg_keys, :arg_values, :status
+        :subworker_class, :superworker_class, :arg_keys, :arg_values, :status, :descendants_are_complete,
+        :meta if ActiveRecord::VERSION::MAJOR < 4
 
-      serialize :children_ids
       serialize :arg_keys
       serialize :arg_values
+      serialize :children_ids
+      serialize :meta
 
       validates_presence_of :subjob_id, :subworker_class, :superworker_class, :status
 

data/lib/generators/sidekiq/superworker/install/templates/create_sidekiq_superworker_subjobs.rb

@@ -13,6 +13,7 @@ class CreateSidekiqSuperworkerSubjobs < ActiveRecord::Migration
       t.text :arg_values
       t.string :status, null: false
       t.boolean :descendants_are_complete, default: false
+      t.text :meta
       
       t.timestamps
     end

data/lib/sidekiq/superworker/dsl_hash.rb

@@ -113,6 +113,7 @@ module Sidekiq
           @records[batch_child_id] = batch_child
 
           @record_id += 1
+          last_subjob_id = nil
           subjobs.values.each_with_index do |subjob, index|
             subjob_id = @record_id
             @record_id += 1
@@ -122,8 +123,9 @@ module Sidekiq
             subjob[:parent_id] = batch_child_id
             subjob[:arg_values] = iteration_args.values
             @records[subjob_id] = subjob
+            @records[last_subjob_id][:next_id] = subjob_id if last_subjob_id
+            last_subjob_id = subjob_id
             nested_hash_to_records_recursive(children, parent_id: subjob_id, scoped_args: iteration_args)
-            subjob[:next_id] = subjob_id + 1 if index < (subjobs.values.length - 1)
           end
 
           children_ids << batch_child_id

data/lib/sidekiq/superworker/subjob_processor.rb

@@ -1,133 +1,140 @@
 module Sidekiq
   module Superworker
     class SubjobProcessor
-      def self.enqueue(subjob)
-        Superworker.debug "#{subjob.to_info}: Trying to enqueue"
-        # Only enqueue subjobs that aren't running, complete, etc
-        return unless subjob.status == 'initialized'
-        
-        Superworker.debug "#{subjob.to_info}: Enqueueing"
-        # If this is a parallel subjob, enqueue all of its children
-        if subjob.subworker_class == 'parallel'
-          subjob.update_attribute(:status, 'running')
-
-          Superworker.debug "#{subjob.to_info}: Enqueueing parallel children"
-          jids = subjob.children.collect do |child|
-            enqueue(child)
-          end
-          jid = jids.first
-        elsif subjob.subworker_class == 'batch'
-          subjob.update_attribute(:status, 'running')
-
-          Superworker.debug "#{subjob.to_info}: Enqueueing batch children"
-          jids = subjob.children.collect do |child|
-            child.update_attribute(:status, 'running')
-            enqueue(child.children.first)
-          end
-          jid = jids.first
-        else
-          klass = "::#{subjob.subworker_class}".constantize
-
-          # If this is a superworker, mark it as complete, which will queue its children or its next subjob
-          if klass.respond_to?(:is_a_superworker?) && klass.is_a_superworker?
-            complete(subjob)
-          # Otherwise, enqueue it in Sidekiq
+      class << self
+        def enqueue(subjob)
+          Superworker.debug "#{subjob.to_info}: Trying to enqueue"
+          # Only enqueue subjobs that aren't running, complete, etc
+          return unless subjob.status == 'initialized'
+          
+          Superworker.debug "#{subjob.to_info}: Enqueueing"
+          # If this is a parallel subjob, enqueue all of its children
+          if subjob.subworker_class == 'parallel'
+            subjob.update_attribute(:status, 'running')
+
+            Superworker.debug "#{subjob.to_info}: Enqueueing parallel children"
+            jids = subjob.children.collect do |child|
+              enqueue(child)
+            end
+            jid = jids.first
+          elsif subjob.subworker_class == 'batch'
+            subjob.update_attribute(:status, 'running')
+
+            Superworker.debug "#{subjob.to_info}: Enqueueing batch children"
+            jids = subjob.children.collect do |child|
+              child.update_attribute(:status, 'running')
+              enqueue(child.children.first)
+            end
+            jid = jids.first
           else
-            # We need to explicitly set the job's JID, so that the ActiveRecord record can be updated before
-            # the job fires off. If the job started first, it could finish before the ActiveRecord update
-            # transaction completes, causing a race condition when finding the ActiveRecord record in
-            # Processor#complete.
-            jid = SecureRandom.hex(12)
-            subjob.update_attributes(
-              jid: jid,
-              status: 'queued'
-            )
-            enqueue_in_sidekiq(subjob, klass, jid)
+            klass = "::#{subjob.subworker_class}".constantize
+
+            # If this is a superworker, mark it as complete, which will queue its children or its next subjob
+            if klass.respond_to?(:is_a_superworker?) && klass.is_a_superworker?
+              complete(subjob)
+            # Otherwise, enqueue it in Sidekiq
+            else
+              # We need to explicitly set the job's JID, so that the ActiveRecord record can be updated before
+              # the job fires off. If the job started first, it could finish before the ActiveRecord update
+              # transaction completes, causing a race condition when finding the ActiveRecord record in
+              # Processor#complete.
+              jid = SecureRandom.hex(12)
+              subjob.update_attributes(
+                jid: jid,
+                status: 'queued'
+              )
+              enqueue_in_sidekiq(subjob, klass, jid)
+            end
           end
+          jid
         end
-        jid
-      end
-
-      def self.enqueue_in_sidekiq(subjob, klass, jid)
-        Superworker.debug "#{subjob.to_info}: Enqueueing in Sidekiq"
 
-        # If sidekiq-unique-jobs is being used for this worker, a number of issues arise if the subjob isn't
-        # queued, so we'll bypass the unique functionality of the worker while running the subjob.
-        is_unique = klass.respond_to?(:sidekiq_options_hash) && !!klass.sidekiq_options_hash['unique']
-        if is_unique
-          unique_value = klass.sidekiq_options_hash.delete('unique')
-          unique_job_expiration_value = klass.sidekiq_options_hash.delete('unique_job_expiration')
-        end
+        def enqueue_in_sidekiq(subjob, klass, jid)
+          Superworker.debug "#{subjob.to_info}: Enqueueing in Sidekiq"
 
-        arg_values = subjob.arg_values
-        # This is akin to perform_async, but it allows us to explicitly set the JID
-        Sidekiq::Client.push('class' => klass, 'args' => arg_values, 'jid' => jid)
+          # If sidekiq-unique-jobs is being used for this worker, a number of issues arise if the subjob isn't
+          # queued, so we'll bypass the unique functionality of the worker while running the subjob.
+          is_unique = klass.respond_to?(:sidekiq_options_hash) && !!(klass.sidekiq_options_hash || {})['unique']
+          if is_unique
+            unique_value = klass.sidekiq_options_hash.delete('unique')
+            unique_job_expiration_value = klass.sidekiq_options_hash.delete('unique_job_expiration')
+          end
 
-        if is_unique
-          klass.sidekiq_options_hash['unique'] = unique_value
-          klass.sidekiq_options_hash['unique_job_expiration'] = unique_job_expiration_value
-        end
+          sidekiq_push(subjob, klass, jid)
 
-        jid
-      end
+          if is_unique
+            klass.sidekiq_options_hash['unique'] = unique_value
+            klass.sidekiq_options_hash['unique_job_expiration'] = unique_job_expiration_value
+          end
 
-      def self.complete(subjob)
-        Superworker.debug "#{subjob.to_info}: Complete"
-        subjob.update_attribute(:status, 'complete')
-
-        # If children are present, enqueue the first one
-        children = subjob.children
-        if children.present?
-          Superworker.debug "#{subjob.to_info}: Enqueueing children"
-          enqueue(children.first)
-          return
-        # Otherwise, set this as having its descendants complete
-        else
-          descendants_are_complete(subjob)
+          jid
         end
-      end
 
-      def self.error(subjob, worker, item, exception)
-        Superworker.debug "#{subjob.to_info}: Error"
-        subjob.update_attribute(:status, 'failed')
-        SuperjobProcessor.error(subjob.superjob_id, worker, item, exception)
-      end
+        def complete(subjob)
+          Superworker.debug "#{subjob.to_info}: Complete"
+          subjob.update_attribute(:status, 'complete')
 
-      def self.descendants_are_complete(subjob)
-        Superworker.debug "#{subjob.to_info}: Descendants are complete"
-        subjob.update_attribute(:descendants_are_complete, true)
+          # If children are present, enqueue the first one
+          children = subjob.children
+          if children.present?
+            Superworker.debug "#{subjob.to_info}: Enqueueing children"
+            enqueue(children.first)
+            return
+          # Otherwise, set this as having its descendants complete
+          else
+            descendants_are_complete(subjob)
+          end
+        end
 
-        if subjob.subworker_class == 'batch_child' || subjob.subworker_class == 'batch'
-          complete(subjob)
+        def error(subjob, worker, item, exception)
+          Superworker.debug "#{subjob.to_info}: Error"
+          subjob.update_attribute(:status, 'failed')
+          SuperjobProcessor.error(subjob.superjob_id, worker, item, exception)
         end
 
-        parent = subjob.parent
-        is_child_of_parallel = parent && parent.subworker_class == 'parallel'
+        protected
+
+        def descendants_are_complete(subjob)
+          Superworker.debug "#{subjob.to_info}: Descendants are complete"
+          subjob.update_attribute(:descendants_are_complete, true)
 
-        # If a parent exists, check whether this subjob's siblings are all complete
-        if parent
-          siblings_descendants_are_complete = parent.children.all? { |child| child.descendants_are_complete }
-          if siblings_descendants_are_complete
-            Superworker.debug "#{subjob.to_info}: Parent (#{parent.to_info}) is complete"
-            descendants_are_complete(parent)
-            parent.update_attribute(:status, 'complete') if is_child_of_parallel
+          if subjob.subworker_class == 'batch_child' || subjob.subworker_class == 'batch'
+            complete(subjob)
           end
-        end
 
-        unless is_child_of_parallel
-          # If a next subjob is present, enqueue it
-          next_subjob = subjob.next
-          if next_subjob
-            enqueue(next_subjob)
-            return
+          parent = subjob.parent
+          is_child_of_parallel = parent && parent.subworker_class == 'parallel'
+
+          # If a parent exists, check whether this subjob's siblings are all complete
+          if parent
+            siblings_descendants_are_complete = parent.children.all? { |child| child.descendants_are_complete }
+            if siblings_descendants_are_complete
+              Superworker.debug "#{subjob.to_info}: Parent (#{parent.to_info}) is complete"
+              descendants_are_complete(parent)
+              parent.update_attribute(:status, 'complete') if is_child_of_parallel
+            end
           end
 
-          # If there isn't a parent, then, this is the final subjob of the superjob
-          unless parent
-            Superworker.debug "#{subjob.to_info}: Superjob is complete"
-            SuperjobProcessor.complete(subjob.superjob_id)
+          unless is_child_of_parallel
+            # If a next subjob is present, enqueue it
+            next_subjob = subjob.next
+            if next_subjob
+              enqueue(next_subjob)
+              return
+            end
+
+            # If there isn't a parent, then, this is the final subjob of the superjob
+            unless parent
+              Superworker.debug "#{subjob.to_info}: Superjob is complete"
+              SuperjobProcessor.complete(subjob.superjob_id)
+            end
           end
         end
+
+        def sidekiq_push(subjob, klass, jid)
+          # This is akin to perform_async, but it allows us to explicitly set the JID
+          Sidekiq::Client.push('class' => klass, 'args' => subjob.arg_values, 'jid' => jid)
+        end
       end
     end
   end

data/lib/sidekiq/superworker/superjob_processor.rb

@@ -8,6 +8,8 @@ module Sidekiq
       def self.create(superjob_id, superworker_class_name, args, subjobs, options={})
         Superworker.debug "Superworker ##{superjob_id}: Create"
         
+        options ||= {}
+        
         # If sidekiq_monitor is being used, create a Sidekiq::Monitor::Job for the superjob
         if defined?(Sidekiq::Monitor)
           now = Time.now

data/lib/sidekiq/superworker/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Superworker
-    VERSION = '0.0.8'
+    VERSION = '0.0.9'
   end
 end

data/lib/sidekiq/superworker/worker.rb

@@ -10,58 +10,11 @@ module Sidekiq
       protected
 
       def self.create_class(class_name, arg_keys, nested_hash)
-        klass = Class.new do
+        klass = Class.new(Sidekiq::Superworker::WorkerClass) do
           @class_name = class_name
           @arg_keys = arg_keys
           @nested_hash = nested_hash
           @dsl_hash = DSLHash.new
-
-          class << self
-            attr_reader :nested_hash
-
-            def is_a_superworker?
-              true
-            end
-
-            def perform_async(*arg_values)
-              superjob_options = {}
-
-              # If an additional argument value is given, it's the superjob's options
-              if (arg_values.length == @arg_keys.length + 1) && arg_values.last.is_a?(Hash)
-                superjob_options = arg_values.last
-              elsif @arg_keys.length != arg_values.length
-                raise "Wrong number of arguments for #{name}.perform_async (#{arg_values.length} for #{@arg_keys.length})"
-              end
-
-              @args = Hash[@arg_keys.zip(arg_values)]
-              @superjob_id = SecureRandom.hex(12)
-              subjobs = create_subjobs(arg_values)
-              SuperjobProcessor.create(@superjob_id, @class_name, arg_values, subjobs, superjob_options)
-            end
-
-            protected
-
-            def create_subjobs(arg_values)
-              records = @dsl_hash.nested_hash_to_records(@nested_hash, @args)
-              subjobs = records.collect do |id, record|
-                record[:status] = 'initialized'
-                record[:superjob_id] = @superjob_id
-                record[:superworker_class] = @class_name
-                unless record.key?(:arg_values)
-                  record[:arg_values] = record[:arg_keys].collect do |arg_key|
-                    # Allow for subjob arg_values to be set within the superworker definition; if a symbol is
-                    # used in the DSL, use @args[arg_key], and otherwise use arg_key as the value
-                    arg_key.is_a?(Symbol) ? @args[arg_key] : arg_key
-                  end
-                end
-                record
-              end
-              # Perform the inserts in a single transaction to improve the performance of large mass inserts
-              Sidekiq::Superworker::Subjob.transaction do
-                Sidekiq::Superworker::Subjob.create(subjobs)
-              end
-            end
-          end
         end
 
         Object.const_set(class_name, klass)

data/lib/sidekiq/superworker/worker_class.rb

@@ -0,0 +1,59 @@
+module Sidekiq
+  module Superworker
+    class WorkerClass
+      class << self
+        attr_reader :nested_hash
+
+        def is_a_superworker?
+          true
+        end
+
+        def perform_async(*arg_values)
+          options = initialize_superjob(arg_values)
+          subjobs = create_subjobs(arg_values)
+          SuperjobProcessor.create(@superjob_id, @class_name, arg_values, subjobs, options)
+        end
+
+        protected
+
+        def initialize_superjob(arg_values)
+          options = nil
+          
+          # If an additional argument value is given, it's the superjob's options
+          if (arg_values.length == @arg_keys.length + 1) && arg_values.last.is_a?(Hash)
+            options = arg_values.last
+          elsif @arg_keys.length != arg_values.length
+            raise "Wrong number of arguments for #{name}.perform_async (#{arg_values.length} for #{@arg_keys.length})"
+          end
+
+          @args = Hash[@arg_keys.zip(arg_values)]
+          @superjob_id = SecureRandom.hex(12)
+
+          options
+        end
+
+        def create_subjobs(arg_values, options={})
+          records = @dsl_hash.nested_hash_to_records(@nested_hash, @args)
+          subjobs = records.collect do |id, record|
+            record[:status] = 'initialized'
+            record[:superjob_id] = @superjob_id
+            record[:superworker_class] = @class_name
+            record[:meta] = options[:meta] if options[:meta]
+            unless record.key?(:arg_values)
+              record[:arg_values] = record[:arg_keys].collect do |arg_key|
+                # Allow for subjob arg_values to be set within the superworker definition; if a symbol is
+                # used in the DSL, use @args[arg_key], and otherwise use arg_key as the value
+                arg_key.is_a?(Symbol) ? @args[arg_key] : arg_key
+              end
+            end
+            record
+          end
+          # Perform the inserts in a single transaction to improve the performance of large mass inserts
+          Sidekiq::Superworker::Subjob.transaction do
+            Sidekiq::Superworker::Subjob.create(subjobs)
+          end
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-superworker
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-20 00:00:00.000000000 Z
+date: 2013-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -113,6 +113,7 @@ files:
 - lib/sidekiq/superworker/superjob_processor.rb
 - lib/sidekiq/superworker/version.rb
 - lib/sidekiq/superworker/worker.rb
+- lib/sidekiq/superworker/worker_class.rb
 - lib/sidekiq/superworker.rb
 - lib/sidekiq-superworker.rb
 - MIT-LICENSE