sidekiq-superworker 0.0.7 → 0.0.8

Sign up to get free protection for your applications and to get access to all the features.
data/README.md CHANGED
@@ -98,6 +98,17 @@ Superworker.create(:MySuperworker, :user_id, :comment_id) do
98
98
  end
99
99
  ```
100
100
 
101
+ ### Logging
102
+
103
+ To make debugging easier, Sidekiq Superworker provides detailed log messages when its logger is set to the DEBUG level:
104
+
105
+ ```ruby
106
+ # config/initializers/superworker.rb
107
+ logger = Logger.new(Rails.root.join('log', 'superworker.log'))
108
+ logger.level = Logger::DEBUG
109
+ Sidekiq::Superworker::Logging.logger = logger
110
+ ```
111
+
101
112
  ### Monitoring
102
113
 
103
114
  Using [sidekiq_monitor](https://github.com/socialpandas/sidekiq_monitor) with Sidekiq Superworker is strongly encouraged, as it lets you easily monitor when a superjob is running, when it has finished, whether it has encountered errors, and the status of all of its subjobs.
@@ -0,0 +1,14 @@
1
+ # Allow a job's JID to be specified in Sidekiq::Client#push
2
+ module Sidekiq
3
+ class Client
4
+ class << self
5
+ alias_method :original_normalize_item, :normalize_item
6
+
7
+ def normalize_item(item)
8
+ normalized_item = original_normalize_item(item)
9
+ normalized_item['jid'] = item['jid'] if item['jid'].present?
10
+ normalized_item
11
+ end
12
+ end
13
+ end
14
+ end
@@ -1,6 +1,7 @@
1
1
  require 'sidekiq'
2
2
 
3
3
  directory = File.dirname(File.absolute_path(__FILE__))
4
+ require "#{directory}/client_ext.rb"
4
5
  Dir.glob("#{directory}/superworker/**/*.rb") { |file| require file }
5
6
  Dir.glob("#{directory}/../generators/sidekiq/superworker/**/*.rb") { |file| require file }
6
7
  Dir.glob("#{directory}/../../app/models/sidekiq/superworker/*.rb") { |file| require file }
@@ -3,15 +3,8 @@ module Sidekiq
3
3
  class Processor
4
4
  def complete(item, new_thread=true)
5
5
  Superworker.debug "JID ##{item['jid']}: Sidekiq job complete"
6
- if new_thread
7
- # Run this in a new thread so that its execution isn't considered to be part of the
8
- # completed job's execution.
9
- Thread.new do
10
- complete_item(item)
11
- end
12
- else
13
- complete_item(item)
14
- end
6
+
7
+ complete_item(item)
15
8
  end
16
9
 
17
10
  def error(worker, item, queue, exception)
@@ -35,19 +28,8 @@ module Sidekiq
35
28
  # Note: The job may've been created outside of sidekiq-superworker, so a nil return value
36
29
  # for this method isn't necessarily problematic
37
30
  def find_subjob_by_jid(jid)
38
- Superworker.debug "JID ##{jid}: Trying to find Subjob"
39
- # The job may complete before the Subjob record is created; in case that happens,
40
- # we need to sleep briefly and requery.
41
- tries = 5
42
- subjob = nil
43
- (1..tries).each do |try|
44
- subjob = Subjob.find_by_jid(jid)
45
- break if subjob
46
- Superworker.debug "JID ##{jid}: Sleeping before trying to find Subjob again"
47
- sleep (2 ** try)
48
- end
49
- Superworker.debug "JID ##{jid}: Subjob found: #{subjob ? subjob.to_info : 'nil'}"
50
- subjob
31
+ Superworker.debug "JID ##{jid}: Finding Subjob"
32
+ Subjob.find_by_jid(jid)
51
33
  end
52
34
  end
53
35
  end
@@ -33,17 +33,22 @@ module Sidekiq
33
33
  complete(subjob)
34
34
  # Otherwise, enqueue it in Sidekiq
35
35
  else
36
- jid = enqueue_in_sidekiq(subjob, klass)
36
+ # We need to explicitly set the job's JID, so that the ActiveRecord record can be updated before
37
+ # the job fires off. If the job started first, it could finish before the ActiveRecord update
38
+ # transaction completes, causing a race condition when finding the ActiveRecord record in
39
+ # Processor#complete.
40
+ jid = SecureRandom.hex(12)
37
41
  subjob.update_attributes(
38
42
  jid: jid,
39
43
  status: 'queued'
40
44
  )
45
+ enqueue_in_sidekiq(subjob, klass, jid)
41
46
  end
42
47
  end
43
48
  jid
44
49
  end
45
50
 
46
- def self.enqueue_in_sidekiq(subjob, klass)
51
+ def self.enqueue_in_sidekiq(subjob, klass, jid)
47
52
  Superworker.debug "#{subjob.to_info}: Enqueueing in Sidekiq"
48
53
 
49
54
  # If sidekiq-unique-jobs is being used for this worker, a number of issues arise if the subjob isn't
@@ -55,8 +60,8 @@ module Sidekiq
55
60
  end
56
61
 
57
62
  arg_values = subjob.arg_values
58
- jid = klass.perform_async(*arg_values)
59
- warn "Nil JID returned by #{subjob.subworker_class}.perform_async with arguments #{arg_values}" if jid.nil?
63
+ # This is akin to perform_async, but it allows us to explicitly set the JID
64
+ Sidekiq::Client.push('class' => klass, 'args' => arg_values, 'jid' => jid)
60
65
 
61
66
  if is_unique
62
67
  klass.sidekiq_options_hash['unique'] = unique_value
@@ -6,7 +6,7 @@ module Sidekiq
6
6
  end
7
7
 
8
8
  def self.create(superjob_id, superworker_class_name, args, subjobs, options={})
9
- Superworker.debug "Superworker ##{superjob_id}: create"
9
+ Superworker.debug "Superworker ##{superjob_id}: Create"
10
10
 
11
11
  # If sidekiq_monitor is being used, create a Sidekiq::Monitor::Job for the superjob
12
12
  if defined?(Sidekiq::Monitor)
@@ -29,7 +29,7 @@ module Sidekiq
29
29
  end
30
30
 
31
31
  def self.complete(superjob_id)
32
- Superworker.debug "Superworker ##{superjob_id}: complete"
32
+ Superworker.debug "Superworker ##{superjob_id}: Complete"
33
33
 
34
34
  # Set the superjob Sidekiq::Monitor::Job as being complete
35
35
  if defined?(Sidekiq::Monitor)
@@ -44,7 +44,7 @@ module Sidekiq
44
44
  end
45
45
 
46
46
  def self.error(superjob_id, worker, item, exception)
47
- Superworker.debug "Superworker ##{superjob_id}: error"
47
+ Superworker.debug "Superworker ##{superjob_id}: Error"
48
48
 
49
49
  if defined?(Sidekiq::Monitor)
50
50
  job = Sidekiq::Monitor::Job.where(queue: queue_name, jid: superjob_id).first
@@ -1,5 +1,5 @@
1
1
  module Sidekiq
2
2
  module Superworker
3
- VERSION = '0.0.7'
3
+ VERSION = '0.0.8'
4
4
  end
5
5
  end
@@ -43,7 +43,7 @@ module Sidekiq
43
43
 
44
44
  def create_subjobs(arg_values)
45
45
  records = @dsl_hash.nested_hash_to_records(@nested_hash, @args)
46
- records.collect do |id, record|
46
+ subjobs = records.collect do |id, record|
47
47
  record[:status] = 'initialized'
48
48
  record[:superjob_id] = @superjob_id
49
49
  record[:superworker_class] = @class_name
@@ -54,7 +54,11 @@ module Sidekiq
54
54
  arg_key.is_a?(Symbol) ? @args[arg_key] : arg_key
55
55
  end
56
56
  end
57
- Sidekiq::Superworker::Subjob.create(record)
57
+ record
58
+ end
59
+ # Perform the inserts in a single transaction to improve the performance of large mass inserts
60
+ Sidekiq::Superworker::Subjob.transaction do
61
+ Sidekiq::Superworker::Subjob.create(subjobs)
58
62
  end
59
63
  end
60
64
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sidekiq-superworker
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.7
4
+ version: 0.0.8
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2013-07-26 00:00:00.000000000 Z
12
+ date: 2013-08-20 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: sidekiq
@@ -102,6 +102,7 @@ files:
102
102
  - app/views/sidekiq/superworker/subjobs.slim
103
103
  - lib/generators/sidekiq/superworker/install/install_generator.rb
104
104
  - lib/generators/sidekiq/superworker/install/templates/create_sidekiq_superworker_subjobs.rb
105
+ - lib/sidekiq/client_ext.rb
105
106
  - lib/sidekiq/superworker/dsl_evaluator.rb
106
107
  - lib/sidekiq/superworker/dsl_hash.rb
107
108
  - lib/sidekiq/superworker/dsl_parser.rb