sidekiq-superworker 0.0.5 → 0.0.6

data/README.md

@@ -53,8 +53,6 @@ Superworker.create(:MySuperworker, :user_id, :comment_id) do
 end
 ```
 
-If you're also using [sidekiq_monitor](https://github.com/socialpandas/sidekiq_monitor), you can easily monitor when a superworker is running and when it has finished.
-
 Installation
 ------------
 
@@ -100,6 +98,45 @@ Superworker.create(:MySuperworker, :user_id, :comment_id) do
 end
 ```
 
+### Monitoring
+
+Using [sidekiq_monitor](https://github.com/socialpandas/sidekiq_monitor) with Sidekiq Superworker is strongly encouraged, as it lets you easily monitor when a superjob is running, when it has finished, whether it has encountered errors, and the status of all of its subjobs.
+
+### Batch Jobs
+
+By using a `batch` block, you can create batches of subjobs that are all associated with the superjob. The following will run Worker1 and Worker2 in serial for every user ID in the array passed to perform_async.
+
+```ruby
+Superworker.create(:MyBatchSuperworker, :user_ids) do
+  batch user_ids: :user_id do
+    Worker1 :user_id
+    Worker2 :user_id
+  end
+end
+
+MyBatchSuperworker.perform_async([30, 31, 32, 33, 34, 35])
+```
+
+Grouping jobs into batches greatly improves your ability to audit them and determine when batches have finished.
+
+### Superjob Names
+
+If you're using sidekiq_monitor and want to set a name for a superjob, you can set it in an additional argument, like so:
+
+```ruby
+# Unnamed
+MySuperworker.perform_async(23)
+
+# Named
+MySuperworker.perform_async(23, name: 'My job name')
+```
+
+### Errors
+
+If a subjob encounters an exception, the subjobs that depend on it won't run, but the rest of the subjobs will continue as usual.
+
+If sidekiq_monitor is being used, the exception will be bubbled up to the superjob, which lets you easily see when your superjobs have failed.
+
 License
 -------
 

data/app/views/sidekiq/superworker/subjobs.slim

@@ -0,0 +1,16 @@
+- subjobs = Sidekiq::Superworker::Subjob.where(superjob_id: job.jid).where('subworker_class != ?', 'batch').order('subjob_id')
+
+table.table.table-condensed.table-striped.table-hover
+  tr
+    th ID
+    th Class
+    th Args
+    th Status
+    th JID
+  - subjobs.each do |subjob|
+    tr
+      td = subjob.id
+      td = subjob.subworker_class
+      td = subjob.arg_values
+      td = subjob.status
+      td = subjob.jid

data/lib/sidekiq/superworker.rb

@@ -28,4 +28,14 @@ Sidekiq.configure_server do |config|
 end
 
 Superworker = Sidekiq::Superworker::Worker unless Object.const_defined?('Superworker')
-Sidekiq::Monitor::Cleaner.add_ignored_queue(Sidekiq::Superworker::SuperjobProcessor.queue_name) if defined?(Sidekiq::Monitor)
+
+# If sidekiq_monitor is being used, customize how superjobs are monitored
+if defined?(Sidekiq::Monitor)
+  # Make Cleaner ignore superjobs, as they don't exist in Redis and thus won't be synced with Sidekiq::Monitor::Job
+  Sidekiq::Monitor::Cleaner.add_ignored_queue(Sidekiq::Superworker::SuperjobProcessor.queue_name) if defined?(Sidekiq::Monitor)
+
+  # Add a custom view that shows the subjobs for a superjob
+  Sidekiq::Monitor::CustomViews.add 'Subjobs', "#{directory}/../../app/views/sidekiq/superworker/subjobs" do |job|
+    job.queue == Sidekiq::Superworker::SuperjobProcessor.queue_name.to_s
+  end
+end

data/lib/sidekiq/superworker/dsl_hash.rb

@@ -0,0 +1,152 @@
+module Sidekiq
+  module Superworker
+    class DSLHash
+      attr_accessor :record_id, :records
+
+      def initialize
+        reset
+      end
+
+      def nested_hash_to_records(nested_hash, args)
+        reset
+        @args = args
+        nested_hash_to_records_recursive(nested_hash)
+      end
+
+      def rewrite_ids_of_nested_hash(nested_hash, record_id)
+        reset
+        @record_id = record_id
+        rewrite_ids_of_nested_hash_recursive(nested_hash)
+      end
+
+      private
+
+      def reset
+        @args = {}
+        @records = {}
+        @record_id = 1
+      end
+
+      def nested_hash_to_records_recursive(nested_hash, options={})
+        return @records if nested_hash.blank?
+
+        defaults = {
+          parent_id: nil,
+          scoped_args: nil # Args that are scoped to this subset of the nested hash (necessary for batch hashes)
+        }
+        options.reverse_merge!(defaults)
+        parent_id = options[:parent_id]
+        last_id = nil
+
+        nested_hash.values.each do |value|
+          id = @record_id
+          @record_id += 1
+          arg_values = value[:arg_keys].collect do |arg_key|
+            # Allow for subjob arg_values to be set within the superworker definition; if a symbol is
+            # used in the DSL, use @args[arg_key], and otherwise use arg_key as the value
+            if arg_key.is_a?(Symbol)
+              options[:scoped_args] ? options[:scoped_args][arg_key] : @args[arg_key]
+            else
+              arg_key
+            end
+          end
+          
+          @records[id] = {
+            subjob_id: id,
+            subworker_class: value[:subworker_class].to_s,
+            arg_keys: value[:arg_keys],
+            arg_values: arg_values,
+            parent_id: parent_id
+          }
+          if value[:subworker_class] == :batch
+            @records[id][:children_ids] = children_ids_for_batch(value[:children], arg_values[0])
+          end
+
+          @records[last_id][:next_id] = id if @records[last_id]
+          last_id = id
+
+          if parent_id && @records[parent_id]
+            @records[parent_id][:children_ids] ||= []
+            @records[parent_id][:children_ids] << id
+          end
+
+          nested_hash_to_records_recursive(value[:children], parent_id: id, scoped_args: options[:scoped_args]) if value[:children] && value[:subworker_class] != :batch
+        end
+        @records
+      end
+
+      def rewrite_ids_of_nested_hash_recursive(nested_hash)
+        new_hash = {}
+        nested_hash.each do |old_record_id, record|
+          @record_id += 1
+          parent_record_id = @record_id
+          new_hash[parent_record_id] = record
+          if record[:children]
+            new_hash[parent_record_id][:children] = rewrite_ids_of_nested_hash_recursive(record[:children])
+          end
+        end
+        new_hash
+      end
+
+      def children_ids_for_batch(subjobs, batch_keys_to_iteration_keys)
+        iteration_keys = batch_keys_to_iteration_keys.values
+        batch_iteration_arg_value_arrays = get_batch_iteration_arg_value_arrays(batch_keys_to_iteration_keys)
+
+        batch_id = @record_id - 1
+
+        children_ids = []
+        batch_iteration_arg_value_arrays.each do |batch_iteration_arg_value_array|
+          iteration_args = {}
+          batch_iteration_arg_value_array.each_with_index do |arg_value, arg_index|
+            arg_key = iteration_keys[arg_index]
+            iteration_args[arg_key] = arg_value
+          end
+
+          batch_child_id = @record_id
+          batch_child = {
+            subjob_id: batch_child_id,
+            subworker_class: 'batch_child',
+            arg_keys: iteration_keys,
+            arg_values: iteration_args.values,
+            parent_id: batch_id
+          }
+          @records[batch_child_id] = batch_child
+
+          @record_id += 1
+          subjobs.values.each_with_index do |subjob, index|
+            subjob_id = @record_id
+            @record_id += 1
+            subjob = subjob.dup
+            children = subjob.delete(:children)
+            subjob[:subjob_id] = subjob_id
+            subjob[:parent_id] = batch_child_id
+            subjob[:arg_values] = iteration_args.values
+            @records[subjob_id] = subjob
+            nested_hash_to_records_recursive(children, parent_id: subjob_id, scoped_args: iteration_args)
+            subjob[:next_id] = subjob_id + 1 if index < (subjobs.values.length - 1)
+          end
+
+          children_ids << batch_child_id
+        end
+        
+        children_ids
+      end
+
+      # Returns an array of argument value arrays, each of which should be passed to each of the
+      # batch iterations
+      def get_batch_iteration_arg_value_arrays(batch_keys_to_iteration_keys)
+        batch_keys = batch_keys_to_iteration_keys.keys
+        batch_keys_to_batch_values = @args.slice(*(batch_keys))
+
+        batch_values = batch_keys_to_batch_values.values
+        first_batch_value = batch_values.pop
+        if batch_values.length > 0
+          batch_values = first_batch_value.zip(batch_values)
+        else
+          batch_values = first_batch_value.zip
+        end
+        batch_values
+      end
+    end
+  end
+end

data/lib/sidekiq/superworker/dsl_parser.rb

@@ -3,75 +3,42 @@ module Sidekiq
     class DSLParser
       def self.parse(block)
         @dsl_evaluator = DSLEvaluator.new
-        set_records_from_block(block)
-        {
-          records: @records,
-          nested_records: @nested_records
-        }
-      end
-
-      def self.set_records_from_block(block)
+        @dsl_hash = DSLHash.new
         @record_id = 0
-        @records = {}
-        @nested_records = block_to_nested_records(block)
-        set_records_from_nested_records(@nested_records)
-      end
-
-      def self.set_records_from_nested_records(nested_records, parent_id=nil)
-        last_id = nil
-        nested_records.each do |id, value|
-          @records[id] = {
-            subjob_id: id,
-            subworker_class: value[:subworker_class].to_s,
-            arg_keys: value[:arg_keys],
-            parent_id: parent_id,
-            children_ids: value[:children] ? value[:children].keys : nil
-          }
-          @records[last_id][:next_id] = id if @records[last_id]
-          last_id = id
-          set_records_from_nested_records(value[:children], id) if value[:children]
-        end
+        block_to_nested_hash(block)
       end
 
-      def self.block_to_nested_records(block)
+      def self.block_to_nested_hash(block)
         fiber = Fiber.new do
           @dsl_evaluator.instance_eval(&block)
         end
         
-        nested_records = {}
+        nested_hash = {}
         while (method_result = fiber.resume)
-          method, arg_keys, block = method_result
+          method, args, block = method_result
           @record_id += 1
           if block
-            nested_records[@record_id] = { subworker_class: method, arg_keys: arg_keys, children: block_to_nested_records(block) }
+            if method == :batch
+              nested_hash[@record_id] = { subworker_class: method, arg_keys: args, children: block_to_nested_hash(block) }
+            else
+              nested_hash[@record_id] = { subworker_class: method, arg_keys: args, children: block_to_nested_hash(block) }
+            end
           else
-            nested_records[@record_id] = { subworker_class: method, arg_keys: arg_keys }
+            nested_hash[@record_id] = { subworker_class: method, arg_keys: args }
           end
 
-          # For superworkers nested within other superworkers, we'll take the subworkers' nested_records,
+          # For superworkers nested within other superworkers, we'll take the subworkers' nested_hash,
           # adjust their ids to fit in with our current @record_id value, and add them into the tree.
-          if method != :parallel
+          unless [:parallel, :batch].include?(method)
             subworker_class = method.to_s.constantize
             if subworker_class.respond_to?(:is_a_superworker?) && subworker_class.is_a_superworker?
               parent_record_id = @record_id
-              nested_records[parent_record_id][:children] = rewrite_ids_of_subworker_records(subworker_class.nested_records)
+              nested_hash[parent_record_id][:children] = @dsl_hash.rewrite_ids_of_nested_hash(subworker_class.nested_hash, @record_id)
+              @record_id = @dsl_hash.record_id
             end
           end
         end
-        nested_records
-      end
-
-      def self.rewrite_ids_of_subworker_records(nested_records)
-        new_hash = {}
-        nested_records.each do |old_record_id, record|
-          @record_id += 1
-          parent_record_id = @record_id
-          new_hash[parent_record_id] = record
-          if record[:children]
-            new_hash[parent_record_id][:children] = rewrite_ids_of_subworker_records(record[:children])
-          end
-        end
-        new_hash
+        nested_hash
       end
     end
   end

data/lib/sidekiq/superworker/subjob_processor.rb

@@ -16,6 +16,15 @@ module Sidekiq
             enqueue(child)
           end
           jid = jids.first
+        elsif subjob.subworker_class == 'batch'
+          subjob.update_attribute(:status, 'running')
+
+          Superworker.debug "#{subjob.to_info}: Enqueueing batch children"
+          jids = subjob.children.collect do |child|
+            child.update_attribute(:status, 'running')
+            enqueue(child.children.first)
+          end
+          jid = jids.first
         else
           klass = "::#{subjob.subworker_class}".constantize
 
@@ -77,11 +86,14 @@ module Sidekiq
         Superworker.debug "#{subjob.to_info}: Descendants are complete"
         subjob.update_attribute(:descendants_are_complete, true)
 
+        if subjob.subworker_class == 'batch_child' || subjob.subworker_class == 'batch'
+          complete(subjob)
+        end
+
         parent = subjob.parent
         is_child_of_parallel = parent && parent.subworker_class == 'parallel'
 
-        # If this is a child of a parallel subjob, check to see if the parent's descendants are all complete
-        # and call descendants_are_complete(parent) if so
+        # If a parent exists, check whether this subjob's siblings are all complete
         if parent
           siblings_descendants_are_complete = parent.children.all? { |child| child.descendants_are_complete }
           if siblings_descendants_are_complete
@@ -99,11 +111,8 @@ module Sidekiq
             return
           end
 
-          # Otherwise, if a parent exists, the parent's descendants are complete
-          if parent
-            descendants_are_complete(parent)
-          # Otherwise, this is the final subjob of the superjob
-          else
+          # If there isn't a parent, then, this is the final subjob of the superjob
+          unless parent
             Superworker.debug "#{subjob.to_info}: Superjob is complete"
             SuperjobProcessor.complete(subjob.superjob_id)
           end

data/lib/sidekiq/superworker/superjob_processor.rb

@@ -5,7 +5,9 @@ module Sidekiq
         :superworker
       end
 
-      def self.create(superjob_id, superworker_class_name, args, subjobs)
+      def self.create(superjob_id, superworker_class_name, args, subjobs, options={})
+        Superworker.debug "Superworker ##{superjob_id}: create"
+        
         # If sidekiq_monitor is being used, create a Sidekiq::Monitor::Job for the superjob
         if defined?(Sidekiq::Monitor)
           now = Time.now
@@ -16,7 +18,8 @@ module Sidekiq
             args: args,
             enqueued_at: now,
             started_at: now,
-            status: 'running'
+            status: 'running',
+            name: options[:name]
           )
         end
 
@@ -26,6 +29,8 @@ module Sidekiq
       end
 
       def self.complete(superjob_id)
+        Superworker.debug "Superworker ##{superjob_id}: complete"
+
         # Set the superjob Sidekiq::Monitor::Job as being complete
         if defined?(Sidekiq::Monitor)
           job = Sidekiq::Monitor::Job.where(queue: queue_name, jid: superjob_id).first
@@ -39,6 +44,8 @@ module Sidekiq
       end
 
       def self.error(superjob_id, worker, item, exception)
+        Superworker.debug "Superworker ##{superjob_id}: error"
+
         if defined?(Sidekiq::Monitor)
           job = Sidekiq::Monitor::Job.where(queue: queue_name, jid: superjob_id).first
           if job

data/lib/sidekiq/superworker/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Superworker
-    VERSION = '0.0.5'
+    VERSION = '0.0.6'
   end
 end

data/lib/sidekiq/superworker/worker.rb

@@ -3,44 +3,56 @@ module Sidekiq
     class Worker
       def self.create(*args, &block)
         class_name = args.shift.to_sym
-        dsl = DSLParser.parse(block)
-        create_class(class_name, args, dsl)
+        nested_hash = DSLParser.parse(block)
+        create_class(class_name, args, nested_hash)
       end
 
       protected
 
-      def self.create_class(class_name, arg_keys, dsl)
+      def self.create_class(class_name, arg_keys, nested_hash)
         klass = Class.new do
           @class_name = class_name
           @arg_keys = arg_keys
-          @records = dsl[:records]
-          @nested_records = dsl[:nested_records]
+          @nested_hash = nested_hash
+          @dsl_hash = DSLHash.new
 
           class << self
-            attr_reader :nested_records
+            attr_reader :nested_hash
 
             def is_a_superworker?
               true
             end
 
             def perform_async(*arg_values)
+              superjob_options = {}
+
+              # If an additional argument value is given, it's the superjob's options
+              if (arg_values.length == @arg_keys.length + 1) && arg_values.last.is_a?(Hash)
+                superjob_options = arg_values.last
+              elsif @arg_keys.length != arg_values.length
+                raise "Wrong number of arguments for #{name}.perform_async (#{arg_values.length} for #{@arg_keys.length})"
+              end
+
               @args = Hash[@arg_keys.zip(arg_values)]
-              subjobs = create_subjobs
-              SuperjobProcessor.create(@superjob_id, @class_name, arg_values, subjobs)
+              @superjob_id = SecureRandom.hex(12)
+              subjobs = create_subjobs(arg_values)
+              SuperjobProcessor.create(@superjob_id, @class_name, arg_values, subjobs, superjob_options)
             end
 
             protected
 
-            def create_subjobs
-              @superjob_id = SecureRandom.hex(12)
-              @records.collect do |id, record|
+            def create_subjobs(arg_values)
+              records = @dsl_hash.nested_hash_to_records(@nested_hash, @args)
+              records.collect do |id, record|
                 record[:status] = 'initialized'
                 record[:superjob_id] = @superjob_id
                 record[:superworker_class] = @class_name
-                record[:arg_values] = record[:arg_keys].collect do |arg_key|
-                  # Allow for subjob arg_values to be set within the superworker definition; if a symbol is
-                  # used in the DSL, use @args[arg_key], and otherwise use arg_key as the value
-                  arg_key.is_a?(Symbol) ? @args[arg_key] : arg_key
+                unless record.key?(:arg_values)
+                  record[:arg_values] = record[:arg_keys].collect do |arg_key|
+                    # Allow for subjob arg_values to be set within the superworker definition; if a symbol is
+                    # used in the DSL, use @args[arg_key], and otherwise use arg_key as the value
+                    arg_key.is_a?(Symbol) ? @args[arg_key] : arg_key
+                  end
                 end
                 Sidekiq::Superworker::Subjob.create(record)
               end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-superworker
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-02 00:00:00.000000000 Z
+date: 2013-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -99,9 +99,11 @@ extensions: []
 extra_rdoc_files: []
 files:
 - app/models/sidekiq/superworker/subjob.rb
+- app/views/sidekiq/superworker/subjobs.slim
 - lib/generators/sidekiq/superworker/install/install_generator.rb
 - lib/generators/sidekiq/superworker/install/templates/create_sidekiq_superworker_subjobs.rb
 - lib/sidekiq/superworker/dsl_evaluator.rb
+- lib/sidekiq/superworker/dsl_hash.rb
 - lib/sidekiq/superworker/dsl_parser.rb
 - lib/sidekiq/superworker/logging.rb
 - lib/sidekiq/superworker/processor.rb