sidekiq-superworker 0.0.7 → 0.0.8

data/README.md

@@ -98,6 +98,17 @@ Superworker.create(:MySuperworker, :user_id, :comment_id) do
 end
 ```
 
+### Logging
+
+To make debugging easier, Sidekiq Superworker provides detailed log messages when its logger is set to the DEBUG level:
+
+```ruby
+# config/initializers/superworker.rb
+logger = Logger.new(Rails.root.join('log', 'superworker.log'))
+logger.level = Logger::DEBUG
+Sidekiq::Superworker::Logging.logger = logger
+```
+
 ### Monitoring
 
 Using [sidekiq_monitor](https://github.com/socialpandas/sidekiq_monitor) with Sidekiq Superworker is strongly encouraged, as it lets you easily monitor when a superjob is running, when it has finished, whether it has encountered errors, and the status of all of its subjobs.

data/lib/sidekiq/client_ext.rb

@@ -0,0 +1,14 @@
+# Allow a job's JID to be specified in Sidekiq::Client#push
+module Sidekiq
+  class Client
+    class << self
+      alias_method :original_normalize_item, :normalize_item
+
+      def normalize_item(item)
+        normalized_item = original_normalize_item(item)
+        normalized_item['jid'] = item['jid'] if item['jid'].present?
+        normalized_item
+      end
+    end
+  end
+end

data/lib/sidekiq/superworker.rb

@@ -1,6 +1,7 @@
 require 'sidekiq'
 
 directory = File.dirname(File.absolute_path(__FILE__))
+require "#{directory}/client_ext.rb"
 Dir.glob("#{directory}/superworker/**/*.rb") { |file| require file }
 Dir.glob("#{directory}/../generators/sidekiq/superworker/**/*.rb") { |file| require file }
 Dir.glob("#{directory}/../../app/models/sidekiq/superworker/*.rb") { |file| require file }

data/lib/sidekiq/superworker/processor.rb

@@ -3,15 +3,8 @@ module Sidekiq
     class Processor
       def complete(item, new_thread=true)
         Superworker.debug "JID ##{item['jid']}: Sidekiq job complete"
-        if new_thread
-          # Run this in a new thread so that its execution isn't considered to be part of the
-          # completed job's execution.
-          Thread.new do
-            complete_item(item)
-          end
-        else
-          complete_item(item)
-        end
+
+        complete_item(item)
       end
 
       def error(worker, item, queue, exception)
@@ -35,19 +28,8 @@ module Sidekiq
       # Note: The job may've been created outside of sidekiq-superworker, so a nil return value
       # for this method isn't necessarily problematic
       def find_subjob_by_jid(jid)
-        Superworker.debug "JID ##{jid}: Trying to find Subjob"
-        # The job may complete before the Subjob record is created; in case that happens,
-        # we need to sleep briefly and requery.
-        tries = 5
-        subjob = nil
-        (1..tries).each do |try|
-          subjob = Subjob.find_by_jid(jid)
-          break if subjob
-          Superworker.debug "JID ##{jid}: Sleeping before trying to find Subjob again"
-          sleep (2 ** try)
-        end
-        Superworker.debug "JID ##{jid}: Subjob found: #{subjob ? subjob.to_info : 'nil'}"
-        subjob
+        Superworker.debug "JID ##{jid}: Finding Subjob"
+        Subjob.find_by_jid(jid)
       end
     end
   end

data/lib/sidekiq/superworker/subjob_processor.rb

@@ -33,17 +33,22 @@ module Sidekiq
             complete(subjob)
           # Otherwise, enqueue it in Sidekiq
           else
-            jid = enqueue_in_sidekiq(subjob, klass)
+            # We need to explicitly set the job's JID, so that the ActiveRecord record can be updated before
+            # the job fires off. If the job started first, it could finish before the ActiveRecord update
+            # transaction completes, causing a race condition when finding the ActiveRecord record in
+            # Processor#complete.
+            jid = SecureRandom.hex(12)
             subjob.update_attributes(
               jid: jid,
               status: 'queued'
             )
+            enqueue_in_sidekiq(subjob, klass, jid)
           end
         end
         jid
       end
 
-      def self.enqueue_in_sidekiq(subjob, klass)
+      def self.enqueue_in_sidekiq(subjob, klass, jid)
         Superworker.debug "#{subjob.to_info}: Enqueueing in Sidekiq"
 
         # If sidekiq-unique-jobs is being used for this worker, a number of issues arise if the subjob isn't
@@ -55,8 +60,8 @@ module Sidekiq
         end
 
         arg_values = subjob.arg_values
-        jid = klass.perform_async(*arg_values)
-        warn "Nil JID returned by #{subjob.subworker_class}.perform_async with arguments #{arg_values}" if jid.nil?
+        # This is akin to perform_async, but it allows us to explicitly set the JID
+        Sidekiq::Client.push('class' => klass, 'args' => arg_values, 'jid' => jid)
 
         if is_unique
           klass.sidekiq_options_hash['unique'] = unique_value

data/lib/sidekiq/superworker/superjob_processor.rb

@@ -6,7 +6,7 @@ module Sidekiq
       end
 
       def self.create(superjob_id, superworker_class_name, args, subjobs, options={})
-        Superworker.debug "Superworker ##{superjob_id}: create"
+        Superworker.debug "Superworker ##{superjob_id}: Create"
         
         # If sidekiq_monitor is being used, create a Sidekiq::Monitor::Job for the superjob
         if defined?(Sidekiq::Monitor)
@@ -29,7 +29,7 @@ module Sidekiq
       end
 
       def self.complete(superjob_id)
-        Superworker.debug "Superworker ##{superjob_id}: complete"
+        Superworker.debug "Superworker ##{superjob_id}: Complete"
 
         # Set the superjob Sidekiq::Monitor::Job as being complete
         if defined?(Sidekiq::Monitor)
@@ -44,7 +44,7 @@ module Sidekiq
       end
 
       def self.error(superjob_id, worker, item, exception)
-        Superworker.debug "Superworker ##{superjob_id}: error"
+        Superworker.debug "Superworker ##{superjob_id}: Error"
 
         if defined?(Sidekiq::Monitor)
           job = Sidekiq::Monitor::Job.where(queue: queue_name, jid: superjob_id).first

data/lib/sidekiq/superworker/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
   module Superworker
-    VERSION = '0.0.7'
+    VERSION = '0.0.8'
   end
 end

data/lib/sidekiq/superworker/worker.rb

@@ -43,7 +43,7 @@ module Sidekiq
 
             def create_subjobs(arg_values)
               records = @dsl_hash.nested_hash_to_records(@nested_hash, @args)
-              records.collect do |id, record|
+              subjobs = records.collect do |id, record|
                 record[:status] = 'initialized'
                 record[:superjob_id] = @superjob_id
                 record[:superworker_class] = @class_name
@@ -54,7 +54,11 @@ module Sidekiq
                     arg_key.is_a?(Symbol) ? @args[arg_key] : arg_key
                   end
                 end
-                Sidekiq::Superworker::Subjob.create(record)
+                record
+              end
+              # Perform the inserts in a single transaction to improve the performance of large mass inserts
+              Sidekiq::Superworker::Subjob.transaction do
+                Sidekiq::Superworker::Subjob.create(subjobs)
               end
             end
           end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-superworker
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-26 00:00:00.000000000 Z
+date: 2013-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -102,6 +102,7 @@ files:
 - app/views/sidekiq/superworker/subjobs.slim
 - lib/generators/sidekiq/superworker/install/install_generator.rb
 - lib/generators/sidekiq/superworker/install/templates/create_sidekiq_superworker_subjobs.rb
+- lib/sidekiq/client_ext.rb
 - lib/sidekiq/superworker/dsl_evaluator.rb
 - lib/sidekiq/superworker/dsl_hash.rb
 - lib/sidekiq/superworker/dsl_parser.rb