sidekiq 3.5.4 → 4.0.0.pre1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/4.0-Upgrade.md +40 -0
- data/Changes.md +5 -8
- data/Ent-Changes.md +6 -0
- data/Gemfile +2 -1
- data/Pro-3.0-Upgrade.md +46 -0
- data/bin/sidekiqctl +3 -3
- data/bin/sidekiqload +17 -6
- data/lib/sidekiq.rb +24 -11
- data/lib/sidekiq/cli.rb +19 -28
- data/lib/sidekiq/client.rb +0 -5
- data/lib/sidekiq/fetch.rb +35 -111
- data/lib/sidekiq/launcher.rb +105 -46
- data/lib/sidekiq/manager.rb +71 -181
- data/lib/sidekiq/middleware/server/retry_jobs.rb +1 -1
- data/lib/sidekiq/processor.rb +119 -96
- data/lib/sidekiq/redis_connection.rb +22 -4
- data/lib/sidekiq/scheduled.rb +47 -26
- data/lib/sidekiq/util.rb +7 -0
- data/lib/sidekiq/version.rb +1 -1
- data/sidekiq.gemspec +1 -1
- data/test/helper.rb +30 -5
- data/test/test_actors.rb +137 -0
- data/test/test_api.rb +395 -394
- data/test/test_fetch.rb +2 -57
- data/test/test_launcher.rb +80 -0
- data/test/test_manager.rb +13 -132
- data/test/test_middleware.rb +3 -5
- data/test/test_processor.rb +20 -57
- data/test/test_scheduled.rb +2 -2
- metadata +19 -14
- data/lib/sidekiq/actor.rb +0 -39
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bd499d714dee9ad44fb3ebeb652cb3f42d15cf28
|
4
|
+
data.tar.gz: 58f3537c50b420df72a7fc3db4cd9b909b26a041
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 431951c7f5d24d3a2fc274c7de7a8576667f542b236ddb704bd99ab5ab94d53ae57153066b3d3e1f674093933a2484ada3044d01a26d8f66f2e08be2447098a5
|
7
|
+
data.tar.gz: 0bbe8c6eeb19eb3f0a9e5cd7ac86f477cc73031c32c03b3e001d272bd04595960e317fe7032f8ee41da83861b236b1f7ffc1b864cfe8c48df67bbe0a1904365b
|
data/4.0-Upgrade.md
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
# Welcome to Sidekiq 4.0!
|
2
|
+
|
3
|
+
Sidekiq 4.0 contains a redesigned, more efficient core with less overhead per job.
|
4
|
+
See my blog for [an overview of Sidekiq 4's higher performance](http://www.mikeperham.com/2015/10/14/optimizing-sidekiq/).
|
5
|
+
|
6
|
+
## What's New
|
7
|
+
|
8
|
+
* Sidekiq no longer uses Celluloid. If your application code uses Celluloid,
|
9
|
+
you will need to pull it in yourself.
|
10
|
+
|
11
|
+
* `redis-namespace` has been removed from Sidekiq's gem dependencies. If
|
12
|
+
you want to use namespacing ([and I strongly urge you not to](http://www.mikeperham.com/2015/09/24/storing-data-with-redis/)), you'll need to add the gem to your Gemfile:
|
13
|
+
```ruby
|
14
|
+
gem 'redis-namespace'
|
15
|
+
```
|
16
|
+
|
17
|
+
* **Redis 2.8.0 or greater is required.** Redis 2.8 was released two years
|
18
|
+
ago and contains **many** useful features which Sidekiq couldn't
|
19
|
+
leverage until now. **Redis 3.0.3 or greater is recommended** for large
|
20
|
+
scale use.
|
21
|
+
|
22
|
+
* Jobs are now fetched from Redis in parallel, making Sidekiq more
|
23
|
+
resilient to high network latency. This means that Sidekiq requires
|
24
|
+
more Redis connections per process. You must have a minimum of
|
25
|
+
`concurrency + 2` connections in your pool or Sidekiq will exit.
|
26
|
+
When in doubt, let Sidekiq size the connection pool for you.
|
27
|
+
|
28
|
+
## Upgrade
|
29
|
+
|
30
|
+
First, make sure you are using Redis 2.8 or greater. Next:
|
31
|
+
|
32
|
+
* Upgrade to the latest Sidekiq 3.x.
|
33
|
+
```ruby
|
34
|
+
gem 'sidekiq', '< 4'
|
35
|
+
```
|
36
|
+
* Fix any deprecation warnings you see.
|
37
|
+
* Upgrade to 4.x.
|
38
|
+
```ruby
|
39
|
+
gem 'sidekiq', '< 5'
|
40
|
+
```
|
data/Changes.md
CHANGED
@@ -1,15 +1,12 @@
|
|
1
1
|
# Sidekiq Changes
|
2
2
|
|
3
|
-
|
3
|
+
4.0.0.pre1
|
4
4
|
-----------
|
5
5
|
|
6
|
-
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
-----------
|
11
|
-
|
12
|
-
- Adjust shutdown event to run in parallel with the rest of system shutdown. [#2635]
|
6
|
+
- Sidekiq's internals have been completely overhauled for performance
|
7
|
+
and to remove dependencies. This has resulted in major speedups, as
|
8
|
+
[detailed on my blog](http://www.mikeperham.com/2015/10/14/optimizing-sidekiq/).
|
9
|
+
- See the [4.0 upgrade notes](4.0-Upgrade.md) for more detail.
|
13
10
|
|
14
11
|
3.5.2
|
15
12
|
-----------
|
data/Ent-Changes.md
CHANGED
@@ -3,6 +3,12 @@ Sidekiq Enterprise Changelog
|
|
3
3
|
|
4
4
|
Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
|
5
5
|
|
6
|
+
HEAD
|
7
|
+
----------
|
8
|
+
|
9
|
+
- Redesign how overrated jobs are rescheduled to avoid creating new
|
10
|
+
jobs. [#2619]
|
11
|
+
|
6
12
|
0.7.5
|
7
13
|
----------
|
8
14
|
|
data/Gemfile
CHANGED
@@ -4,12 +4,12 @@ gemspec
|
|
4
4
|
gem 'rails', '~> 4.2'
|
5
5
|
gem 'simplecov'
|
6
6
|
gem 'minitest'
|
7
|
+
gem 'minitest-utils'
|
7
8
|
gem 'toxiproxy'
|
8
9
|
|
9
10
|
platforms :rbx do
|
10
11
|
gem 'rubysl', '~> 2.0' # if using anything in the ruby standard library
|
11
12
|
gem 'psych' # if using yaml
|
12
|
-
gem 'minitest' # if using minitest
|
13
13
|
gem 'rubinius-developer_tools' # if using any of coverage, debugger, profiler
|
14
14
|
end
|
15
15
|
|
@@ -19,6 +19,7 @@ end
|
|
19
19
|
|
20
20
|
platforms :mri do
|
21
21
|
gem 'pry-byebug'
|
22
|
+
gem 'ruby-prof'
|
22
23
|
end
|
23
24
|
|
24
25
|
platforms :jruby do
|
data/Pro-3.0-Upgrade.md
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
# Welcome to Sidekiq Pro 3.0!
|
2
|
+
|
3
|
+
Sidekiq Pro 3.0 is designed to work with Sidekiq 4.0.
|
4
|
+
|
5
|
+
## What's New
|
6
|
+
|
7
|
+
* **Redis 2.8.0 or greater is required.** Redis 2.8 was released two years
|
8
|
+
ago and contains **many** useful features which Sidekiq couldn't
|
9
|
+
leverage until now. **Redis 3.0.3 or greater is recommended** for large
|
10
|
+
scale use.
|
11
|
+
|
12
|
+
* Sidekiq Pro no longer uses Celluloid. If your application code uses Celluloid,
|
13
|
+
you will need to pull it in yourself.
|
14
|
+
|
15
|
+
* Pausing and unpausing queues is now instantaneous, no more polling!
|
16
|
+
|
17
|
+
* Reliable fetch has been re-implemented due to the fetch changes in
|
18
|
+
Sidekiq 4.0.
|
19
|
+
|
20
|
+
* Support for platforms without persistent hostnames. Since reliable fetch
|
21
|
+
normally requires a persistent hostname, you may disable hostname usage on
|
22
|
+
platforms like Heroku and Docker:
|
23
|
+
```ruby
|
24
|
+
Sidekiq.configure_server do |config|
|
25
|
+
config.options[:ephemeral_hostname] = true
|
26
|
+
config.reliable_fetch!
|
27
|
+
end
|
28
|
+
```
|
29
|
+
This option is enabled automatically if Heroku's DYNO environment variable is present.
|
30
|
+
Without a persistent hostname, each Sidekiq process **must** have its own unique index.
|
31
|
+
|
32
|
+
* The old 'sidekiq/notifications' features have been removed.
|
33
|
+
|
34
|
+
## Upgrade
|
35
|
+
|
36
|
+
First, make sure you are using Redis 2.8 or greater. Next:
|
37
|
+
|
38
|
+
* Upgrade to the latest Sidekiq Pro 2.x.
|
39
|
+
```ruby
|
40
|
+
gem 'sidekiq-pro', '< 3'
|
41
|
+
```
|
42
|
+
* Fix any deprecation warnings you see.
|
43
|
+
* Upgrade to 3.x.
|
44
|
+
```ruby
|
45
|
+
gem 'sidekiq-pro', '< 4'
|
46
|
+
```
|
data/bin/sidekiqctl
CHANGED
@@ -60,11 +60,11 @@ class Sidekiqctl
|
|
60
60
|
end
|
61
61
|
|
62
62
|
def quiet
|
63
|
-
|
63
|
+
Process.kill(:USR1, pid)
|
64
64
|
end
|
65
65
|
|
66
66
|
def stop
|
67
|
-
|
67
|
+
Process.kill(:TERM, pid)
|
68
68
|
kill_timeout.times do
|
69
69
|
begin
|
70
70
|
Process.getpgid(pid)
|
@@ -74,7 +74,7 @@ class Sidekiqctl
|
|
74
74
|
end
|
75
75
|
sleep 1
|
76
76
|
end
|
77
|
-
|
77
|
+
Process.kill(:KILL, pid)
|
78
78
|
FileUtils.rm_f pidfile
|
79
79
|
done 'Sidekiq shut down forcefully.'
|
80
80
|
end
|
data/bin/sidekiqload
CHANGED
@@ -3,18 +3,30 @@
|
|
3
3
|
# Quiet some warnings we see when running in warning mode:
|
4
4
|
# RUBYOPT=-w bundle exec sidekiq
|
5
5
|
$TESTING = false
|
6
|
-
$CELLULOID_DEBUG = false
|
7
6
|
|
8
|
-
require '
|
9
|
-
|
7
|
+
#require 'ruby-prof'
|
8
|
+
|
10
9
|
require_relative '../lib/sidekiq/cli'
|
11
10
|
require_relative '../lib/sidekiq/launcher'
|
12
|
-
Celluloid.logger = nil
|
13
11
|
|
14
12
|
include Sidekiq::Util
|
15
13
|
|
14
|
+
# brew tap shopify/shopify
|
15
|
+
# brew install toxiproxy
|
16
|
+
# gem install toxiproxy
|
17
|
+
require 'toxiproxy'
|
18
|
+
# simulate a non-localhost network for realer-world conditions.
|
19
|
+
# adding 1ms of network latency has an ENORMOUS impact on benchmarks
|
20
|
+
Toxiproxy.populate([{
|
21
|
+
"name": "redis",
|
22
|
+
"listen": "127.0.0.1:6380",
|
23
|
+
"upstream": "127.0.0.1:6379"
|
24
|
+
}])
|
25
|
+
|
26
|
+
|
16
27
|
Sidekiq.configure_server do |config|
|
17
28
|
config.redis = { db: 13, port: 6380 }
|
29
|
+
#config.redis = { db: 13 }
|
18
30
|
config.options[:queues] << 'default'
|
19
31
|
config.logger.level = Logger::ERROR
|
20
32
|
config.average_scheduled_poll_interval = 2
|
@@ -68,8 +80,7 @@ def handle_signal(launcher, sig)
|
|
68
80
|
raise Interrupt
|
69
81
|
when 'USR1'
|
70
82
|
Sidekiq.logger.info "Received USR1, no longer accepting new work"
|
71
|
-
launcher.
|
72
|
-
#fire_event(:quiet, true)
|
83
|
+
launcher.quiet
|
73
84
|
when 'USR2'
|
74
85
|
if Sidekiq.options[:logfile]
|
75
86
|
Sidekiq.logger.info "Received USR2, reopening log file"
|
data/lib/sidekiq.rb
CHANGED
@@ -110,11 +110,25 @@ module Sidekiq
|
|
110
110
|
end
|
111
111
|
|
112
112
|
def self.server_middleware
|
113
|
-
@server_chain ||=
|
113
|
+
@server_chain ||= default_server_middleware
|
114
114
|
yield @server_chain if block_given?
|
115
115
|
@server_chain
|
116
116
|
end
|
117
117
|
|
118
|
+
def self.default_server_middleware
|
119
|
+
require 'sidekiq/middleware/server/retry_jobs'
|
120
|
+
require 'sidekiq/middleware/server/logging'
|
121
|
+
|
122
|
+
Middleware::Chain.new do |m|
|
123
|
+
m.add Middleware::Server::Logging
|
124
|
+
m.add Middleware::Server::RetryJobs
|
125
|
+
if defined?(::ActiveRecord::Base)
|
126
|
+
require 'sidekiq/middleware/server/active_record'
|
127
|
+
m.add Sidekiq::Middleware::Server::ActiveRecord
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
118
132
|
def self.default_worker_options=(hash)
|
119
133
|
@default_worker_options = default_worker_options.merge(hash.stringify_keys)
|
120
134
|
end
|
@@ -139,16 +153,6 @@ module Sidekiq
|
|
139
153
|
Sidekiq::Logging.logger = log
|
140
154
|
end
|
141
155
|
|
142
|
-
# When set, overrides Sidekiq.options[:average_scheduled_poll_interval] and sets
|
143
|
-
# the average interval that this process will delay before checking for
|
144
|
-
# scheduled jobs or job retries that are ready to run.
|
145
|
-
#
|
146
|
-
# See sidekiq/scheduled.rb for an in-depth explanation of this value
|
147
|
-
def self.poll_interval=(interval)
|
148
|
-
$stderr.puts "DEPRECATION: `config.poll_interval = #{interval}` will be removed in Sidekiq 4. Please update to `config.average_scheduled_poll_interval = #{interval}`."
|
149
|
-
self.options[:poll_interval_average] = interval
|
150
|
-
end
|
151
|
-
|
152
156
|
# How frequently Redis should be checked by a random Sidekiq process for
|
153
157
|
# scheduled and retriable jobs. Each individual process will take turns by
|
154
158
|
# waiting some multiple of this value.
|
@@ -182,6 +186,15 @@ module Sidekiq
|
|
182
186
|
raise ArgumentError, "Invalid event name: #{event}" unless options[:lifecycle_events].key?(event)
|
183
187
|
options[:lifecycle_events][event] << block
|
184
188
|
end
|
189
|
+
|
190
|
+
# We are shutting down Sidekiq but what about workers that
|
191
|
+
# are working on some long job? This error is
|
192
|
+
# raised in workers that have not finished within the hard
|
193
|
+
# timeout limit. This is needed to rollback db transactions,
|
194
|
+
# otherwise Ruby's Thread#kill will commit. See #377.
|
195
|
+
# DO NOT RESCUE THIS ERROR IN YOUR WORKERS
|
196
|
+
class Shutdown < Interrupt; end
|
197
|
+
|
185
198
|
end
|
186
199
|
|
187
200
|
require 'sidekiq/extensions/class_methods'
|
data/lib/sidekiq/cli.rb
CHANGED
@@ -11,18 +11,18 @@ require 'sidekiq'
|
|
11
11
|
require 'sidekiq/util'
|
12
12
|
|
13
13
|
module Sidekiq
|
14
|
-
# We are shutting down Sidekiq but what about workers that
|
15
|
-
# are working on some long job? This error is
|
16
|
-
# raised in workers that have not finished within the hard
|
17
|
-
# timeout limit. This is needed to rollback db transactions,
|
18
|
-
# otherwise Ruby's Thread#kill will commit. See #377.
|
19
|
-
# DO NOT RESCUE THIS ERROR.
|
20
|
-
class Shutdown < Interrupt; end
|
21
|
-
|
22
14
|
class CLI
|
23
15
|
include Util
|
24
16
|
include Singleton unless $TESTING
|
25
17
|
|
18
|
+
PROCTITLES = [
|
19
|
+
proc { 'sidekiq'.freeze },
|
20
|
+
proc { Sidekiq::VERSION },
|
21
|
+
proc { |me, data| data['tag'] },
|
22
|
+
proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
|
23
|
+
proc { |me, data| "stopping" if me.stopping? },
|
24
|
+
]
|
25
|
+
|
26
26
|
# Used for CLI testing
|
27
27
|
attr_accessor :code
|
28
28
|
attr_accessor :launcher
|
@@ -40,7 +40,6 @@ module Sidekiq
|
|
40
40
|
validate!
|
41
41
|
daemonize
|
42
42
|
write_pid
|
43
|
-
load_celluloid
|
44
43
|
end
|
45
44
|
|
46
45
|
# Code within this method is not tested because it alters
|
@@ -66,17 +65,21 @@ module Sidekiq
|
|
66
65
|
logger.info Sidekiq::LICENSE
|
67
66
|
logger.info "Upgrade to Sidekiq Pro for more features and support: http://sidekiq.org" unless defined?(::Sidekiq::Pro)
|
68
67
|
|
68
|
+
Sidekiq.redis do |conn|
|
69
|
+
# touch the connection pool so it is created before we
|
70
|
+
# fire startup and start multithreading.
|
71
|
+
ver = conn.info['redis_version']
|
72
|
+
raise "Oops, you are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
|
73
|
+
end
|
74
|
+
|
75
|
+
# Before this point, the process is initializing with just the main thread.
|
76
|
+
# Starting here the process will now have multiple threads running.
|
69
77
|
fire_event(:startup)
|
70
78
|
|
71
79
|
logger.debug {
|
72
80
|
"Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}"
|
73
81
|
}
|
74
82
|
|
75
|
-
Sidekiq.redis do |conn|
|
76
|
-
# touch the connection pool so it is created before we
|
77
|
-
# launch the actors.
|
78
|
-
end
|
79
|
-
|
80
83
|
if !options[:daemon]
|
81
84
|
logger.info 'Starting processing, hit Ctrl-C to stop'
|
82
85
|
end
|
@@ -94,6 +97,7 @@ module Sidekiq
|
|
94
97
|
rescue Interrupt
|
95
98
|
logger.info 'Shutting down'
|
96
99
|
launcher.stop
|
100
|
+
fire_event(:shutdown, true)
|
97
101
|
# Explicitly exit so busy Processor threads can't block
|
98
102
|
# process shutdown.
|
99
103
|
exit(0)
|
@@ -129,7 +133,7 @@ module Sidekiq
|
|
129
133
|
raise Interrupt
|
130
134
|
when 'USR1'
|
131
135
|
Sidekiq.logger.info "Received USR1, no longer accepting new work"
|
132
|
-
launcher.
|
136
|
+
launcher.quiet
|
133
137
|
fire_event(:quiet, true)
|
134
138
|
when 'USR2'
|
135
139
|
if Sidekiq.options[:logfile]
|
@@ -159,19 +163,6 @@ module Sidekiq
|
|
159
163
|
end
|
160
164
|
end
|
161
165
|
|
162
|
-
def load_celluloid
|
163
|
-
raise "Celluloid cannot be required until here, or it will break Sidekiq's daemonization" if defined?(::Celluloid) && options[:daemon]
|
164
|
-
|
165
|
-
# Celluloid can't be loaded until after we've daemonized
|
166
|
-
# because it spins up threads and creates locks which get
|
167
|
-
# into a very bad state if forked.
|
168
|
-
require 'celluloid/current'
|
169
|
-
Celluloid.logger = (options[:verbose] ? Sidekiq.logger : nil)
|
170
|
-
|
171
|
-
require 'sidekiq/manager'
|
172
|
-
require 'sidekiq/scheduled'
|
173
|
-
end
|
174
|
-
|
175
166
|
def daemonize
|
176
167
|
return unless options[:daemon]
|
177
168
|
|
data/lib/sidekiq/client.rb
CHANGED
data/lib/sidekiq/fetch.rb
CHANGED
@@ -1,101 +1,34 @@
|
|
1
1
|
require 'sidekiq'
|
2
|
-
require 'sidekiq/util'
|
3
|
-
require 'sidekiq/actor'
|
4
2
|
|
5
3
|
module Sidekiq
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
class Fetcher
|
11
|
-
include Util
|
12
|
-
include Actor
|
13
|
-
|
14
|
-
TIMEOUT = 1
|
15
|
-
|
16
|
-
attr_reader :down
|
17
|
-
|
18
|
-
def initialize(mgr, options)
|
19
|
-
@down = nil
|
20
|
-
@mgr = mgr
|
21
|
-
@strategy = Fetcher.strategy.new(options)
|
22
|
-
end
|
23
|
-
|
24
|
-
# Fetching is straightforward: the Manager makes a fetch
|
25
|
-
# request for each idle processor when Sidekiq starts and
|
26
|
-
# then issues a new fetch request every time a Processor
|
27
|
-
# finishes a message.
|
28
|
-
#
|
29
|
-
# Because we have to shut down cleanly, we can't block
|
30
|
-
# forever and we can't loop forever. Instead we reschedule
|
31
|
-
# a new fetch if the current fetch turned up nothing.
|
32
|
-
def fetch
|
33
|
-
watchdog('Fetcher#fetch died') do
|
34
|
-
return if Sidekiq::Fetcher.done?
|
35
|
-
|
36
|
-
begin
|
37
|
-
work = @strategy.retrieve_work
|
38
|
-
::Sidekiq.logger.info("Redis is online, #{Time.now - @down} sec downtime") if @down
|
39
|
-
@down = nil
|
40
|
-
|
41
|
-
if work
|
42
|
-
@mgr.async.assign(work)
|
43
|
-
else
|
44
|
-
after(0) { fetch }
|
45
|
-
end
|
46
|
-
rescue => ex
|
47
|
-
handle_fetch_exception(ex)
|
48
|
-
end
|
4
|
+
class BasicFetch
|
5
|
+
# We want the fetch operation to timeout every few seconds so the thread
|
6
|
+
# can check if the process is shutting down.
|
7
|
+
TIMEOUT = 2
|
49
8
|
|
9
|
+
UnitOfWork = Struct.new(:queue, :job) do
|
10
|
+
def acknowledge
|
11
|
+
# nothing to do
|
50
12
|
end
|
51
|
-
end
|
52
|
-
|
53
|
-
private
|
54
13
|
|
55
|
-
|
56
|
-
|
57
|
-
|
14
|
+
def queue_name
|
15
|
+
queue.gsub(/.*queue:/, ''.freeze)
|
16
|
+
end
|
58
17
|
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
ex.backtrace.each do |bt|
|
63
|
-
logger.error(bt)
|
18
|
+
def requeue
|
19
|
+
Sidekiq.redis do |conn|
|
20
|
+
conn.rpush("queue:#{queue_name}", job)
|
64
21
|
end
|
65
22
|
end
|
66
|
-
@down ||= Time.now
|
67
|
-
pause
|
68
|
-
after(0) { fetch }
|
69
|
-
rescue Celluloid::TaskTerminated
|
70
|
-
# If redis is down when we try to shut down, all the fetch backlog
|
71
|
-
# raises these errors. Haven't been able to figure out what I'm doing wrong.
|
72
|
-
end
|
73
|
-
|
74
|
-
# Ugh. Say hello to a bloody hack.
|
75
|
-
# Can't find a clean way to get the fetcher to just stop processing
|
76
|
-
# its mailbox when shutdown starts.
|
77
|
-
def self.done!
|
78
|
-
@done = true
|
79
|
-
end
|
80
|
-
|
81
|
-
def self.reset # testing only
|
82
|
-
@done = nil
|
83
23
|
end
|
84
24
|
|
85
|
-
def self.done?
|
86
|
-
defined?(@done) && @done
|
87
|
-
end
|
88
|
-
|
89
|
-
def self.strategy
|
90
|
-
Sidekiq.options[:fetch] || BasicFetch
|
91
|
-
end
|
92
|
-
end
|
93
|
-
|
94
|
-
class BasicFetch
|
95
25
|
def initialize(options)
|
96
26
|
@strictly_ordered_queues = !!options[:strict]
|
97
27
|
@queues = options[:queues].map { |q| "queue:#{q}" }
|
98
|
-
|
28
|
+
if @strictly_ordered_queues
|
29
|
+
@queues = @queues.uniq
|
30
|
+
@queues << TIMEOUT
|
31
|
+
end
|
99
32
|
end
|
100
33
|
|
101
34
|
def retrieve_work
|
@@ -103,6 +36,22 @@ module Sidekiq
|
|
103
36
|
UnitOfWork.new(*work) if work
|
104
37
|
end
|
105
38
|
|
39
|
+
# Creating the Redis#brpop command takes into account any
|
40
|
+
# configured queue weights. By default Redis#brpop returns
|
41
|
+
# data from the first queue that has pending elements. We
|
42
|
+
# recreate the queue command each time we invoke Redis#brpop
|
43
|
+
# to honor weights and avoid queue starvation.
|
44
|
+
def queues_cmd
|
45
|
+
if @strictly_ordered_queues
|
46
|
+
@queues
|
47
|
+
else
|
48
|
+
queues = @queues.shuffle.uniq
|
49
|
+
queues << TIMEOUT
|
50
|
+
queues
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
|
106
55
|
# By leaving this as a class method, it can be pluggable and used by the Manager actor. Making it
|
107
56
|
# an instance method will make it async to the Fetcher actor
|
108
57
|
def self.bulk_requeue(inprogress, options)
|
@@ -112,7 +61,7 @@ module Sidekiq
|
|
112
61
|
jobs_to_requeue = {}
|
113
62
|
inprogress.each do |unit_of_work|
|
114
63
|
jobs_to_requeue[unit_of_work.queue_name] ||= []
|
115
|
-
jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.
|
64
|
+
jobs_to_requeue[unit_of_work.queue_name] << unit_of_work.job
|
116
65
|
end
|
117
66
|
|
118
67
|
Sidekiq.redis do |conn|
|
@@ -122,35 +71,10 @@ module Sidekiq
|
|
122
71
|
end
|
123
72
|
end
|
124
73
|
end
|
125
|
-
Sidekiq.logger.info("Pushed #{inprogress.size}
|
74
|
+
Sidekiq.logger.info("Pushed #{inprogress.size} jobs back to Redis")
|
126
75
|
rescue => ex
|
127
76
|
Sidekiq.logger.warn("Failed to requeue #{inprogress.size} jobs: #{ex.message}")
|
128
77
|
end
|
129
78
|
|
130
|
-
UnitOfWork = Struct.new(:queue, :message) do
|
131
|
-
def acknowledge
|
132
|
-
# nothing to do
|
133
|
-
end
|
134
|
-
|
135
|
-
def queue_name
|
136
|
-
queue.gsub(/.*queue:/, '')
|
137
|
-
end
|
138
|
-
|
139
|
-
def requeue
|
140
|
-
Sidekiq.redis do |conn|
|
141
|
-
conn.rpush("queue:#{queue_name}", message)
|
142
|
-
end
|
143
|
-
end
|
144
|
-
end
|
145
|
-
|
146
|
-
# Creating the Redis#brpop command takes into account any
|
147
|
-
# configured queue weights. By default Redis#brpop returns
|
148
|
-
# data from the first queue that has pending elements. We
|
149
|
-
# recreate the queue command each time we invoke Redis#brpop
|
150
|
-
# to honor weights and avoid queue starvation.
|
151
|
-
def queues_cmd
|
152
|
-
queues = @strictly_ordered_queues ? @unique_queues.dup : @queues.shuffle.uniq
|
153
|
-
queues << Sidekiq::Fetcher::TIMEOUT
|
154
|
-
end
|
155
79
|
end
|
156
80
|
end
|