sidekiq 4.2.4 → 5.2.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/.github/issue_template.md +8 -1
- data/.gitignore +1 -0
- data/.travis.yml +5 -3
- data/5.0-Upgrade.md +56 -0
- data/COMM-LICENSE +1 -1
- data/Changes.md +151 -0
- data/Ent-Changes.md +77 -2
- data/Gemfile +10 -25
- data/LICENSE +1 -1
- data/Pro-4.0-Upgrade.md +35 -0
- data/Pro-Changes.md +156 -2
- data/README.md +9 -6
- data/Rakefile +1 -2
- data/bin/sidekiqctl +1 -1
- data/bin/sidekiqload +15 -33
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +1 -1
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +1 -1
- data/lib/sidekiq/api.rb +157 -67
- data/lib/sidekiq/cli.rb +71 -26
- data/lib/sidekiq/client.rb +25 -18
- data/lib/sidekiq/core_ext.rb +1 -106
- data/lib/sidekiq/delay.rb +42 -0
- data/lib/sidekiq/exception_handler.rb +2 -4
- data/lib/sidekiq/extensions/generic_proxy.rb +7 -1
- data/lib/sidekiq/fetch.rb +1 -1
- data/lib/sidekiq/job_logger.rb +25 -0
- data/lib/sidekiq/job_retry.rb +241 -0
- data/lib/sidekiq/launcher.rb +45 -37
- data/lib/sidekiq/logging.rb +18 -2
- data/lib/sidekiq/manager.rb +3 -4
- data/lib/sidekiq/middleware/server/active_record.rb +10 -0
- data/lib/sidekiq/processor.rb +91 -34
- data/lib/sidekiq/rails.rb +15 -51
- data/lib/sidekiq/redis_connection.rb +31 -5
- data/lib/sidekiq/scheduled.rb +35 -8
- data/lib/sidekiq/testing.rb +24 -7
- data/lib/sidekiq/util.rb +6 -2
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +2 -6
- data/lib/sidekiq/web/application.rb +28 -21
- data/lib/sidekiq/web/helpers.rb +67 -23
- data/lib/sidekiq/web/router.rb +14 -10
- data/lib/sidekiq/web.rb +4 -4
- data/lib/sidekiq/worker.rb +97 -14
- data/lib/sidekiq.rb +23 -24
- data/sidekiq.gemspec +7 -10
- data/web/assets/javascripts/application.js +0 -0
- data/web/assets/javascripts/dashboard.js +18 -13
- data/web/assets/stylesheets/application-rtl.css +246 -0
- data/web/assets/stylesheets/application.css +336 -4
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +80 -0
- data/web/locales/en.yml +1 -0
- data/web/locales/es.yml +4 -3
- data/web/locales/fa.yml +80 -0
- data/web/locales/he.yml +79 -0
- data/web/locales/ja.yml +5 -3
- data/web/locales/ur.yml +80 -0
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +1 -1
- data/web/views/_nav.erb +1 -1
- data/web/views/_paging.erb +1 -1
- data/web/views/busy.erb +9 -5
- data/web/views/dashboard.erb +3 -3
- data/web/views/layout.erb +11 -2
- data/web/views/morgue.erb +14 -10
- data/web/views/queue.erb +10 -10
- data/web/views/queues.erb +4 -2
- data/web/views/retries.erb +13 -11
- data/web/views/retry.erb +1 -1
- data/web/views/scheduled.erb +2 -2
- metadata +26 -160
- data/lib/sidekiq/middleware/server/logging.rb +0 -40
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -205
- data/test/config.yml +0 -9
- data/test/env_based_config.yml +0 -11
- data/test/fake_env.rb +0 -1
- data/test/fixtures/en.yml +0 -2
- data/test/helper.rb +0 -75
- data/test/test_actors.rb +0 -138
- data/test/test_api.rb +0 -528
- data/test/test_cli.rb +0 -418
- data/test/test_client.rb +0 -266
- data/test/test_exception_handler.rb +0 -56
- data/test/test_extensions.rb +0 -127
- data/test/test_fetch.rb +0 -50
- data/test/test_launcher.rb +0 -95
- data/test/test_logging.rb +0 -35
- data/test/test_manager.rb +0 -50
- data/test/test_middleware.rb +0 -158
- data/test/test_processor.rb +0 -235
- data/test/test_rails.rb +0 -22
- data/test/test_redis_connection.rb +0 -132
- data/test/test_retry.rb +0 -326
- data/test/test_retry_exhausted.rb +0 -149
- data/test/test_scheduled.rb +0 -115
- data/test/test_scheduling.rb +0 -58
- data/test/test_sidekiq.rb +0 -107
- data/test/test_testing.rb +0 -143
- data/test/test_testing_fake.rb +0 -357
- data/test/test_testing_inline.rb +0 -94
- data/test/test_util.rb +0 -13
- data/test/test_web.rb +0 -726
- data/test/test_web_helpers.rb +0 -54
data/test/test_processor.rb
DELETED
@@ -1,235 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
require_relative 'helper'
|
3
|
-
require 'sidekiq/fetch'
|
4
|
-
require 'sidekiq/cli'
|
5
|
-
require 'sidekiq/processor'
|
6
|
-
|
7
|
-
class TestProcessor < Sidekiq::Test
|
8
|
-
TestException = Class.new(StandardError)
|
9
|
-
TEST_EXCEPTION = TestException.new("kerboom!")
|
10
|
-
|
11
|
-
describe 'processor' do
|
12
|
-
before do
|
13
|
-
$invokes = 0
|
14
|
-
@mgr = Minitest::Mock.new
|
15
|
-
@mgr.expect(:options, {:queues => ['default']})
|
16
|
-
@mgr.expect(:options, {:queues => ['default']})
|
17
|
-
@processor = ::Sidekiq::Processor.new(@mgr)
|
18
|
-
end
|
19
|
-
|
20
|
-
class MockWorker
|
21
|
-
include Sidekiq::Worker
|
22
|
-
def perform(args)
|
23
|
-
raise TEST_EXCEPTION if args == 'boom'
|
24
|
-
args.pop if args.is_a? Array
|
25
|
-
$invokes += 1
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
def work(msg, queue='queue:default')
|
30
|
-
Sidekiq::BasicFetch::UnitOfWork.new(queue, msg)
|
31
|
-
end
|
32
|
-
|
33
|
-
it 'processes as expected' do
|
34
|
-
msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['myarg'] })
|
35
|
-
@processor.process(work(msg))
|
36
|
-
assert_equal 1, $invokes
|
37
|
-
end
|
38
|
-
|
39
|
-
it 'executes a worker as expected' do
|
40
|
-
worker = Minitest::Mock.new
|
41
|
-
worker.expect(:perform, nil, [1, 2, 3])
|
42
|
-
@processor.execute_job(worker, [1, 2, 3])
|
43
|
-
end
|
44
|
-
|
45
|
-
it 're-raises exceptions after handling' do
|
46
|
-
msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
|
47
|
-
re_raise = false
|
48
|
-
|
49
|
-
begin
|
50
|
-
@processor.process(work(msg))
|
51
|
-
flunk "Expected exception"
|
52
|
-
rescue TestException
|
53
|
-
re_raise = true
|
54
|
-
end
|
55
|
-
|
56
|
-
assert_equal 0, $invokes
|
57
|
-
assert re_raise, "does not re-raise exceptions after handling"
|
58
|
-
end
|
59
|
-
|
60
|
-
it 'does not modify original arguments' do
|
61
|
-
msg = { 'class' => MockWorker.to_s, 'args' => [['myarg']] }
|
62
|
-
msgstr = Sidekiq.dump_json(msg)
|
63
|
-
@mgr.expect(:processor_done, nil, [@processor])
|
64
|
-
@processor.process(work(msgstr))
|
65
|
-
assert_equal [['myarg']], msg['args']
|
66
|
-
end
|
67
|
-
|
68
|
-
describe 'exception handling' do
|
69
|
-
let(:errors) { [] }
|
70
|
-
let(:error_handler) { proc { |ex, _| errors << ex } }
|
71
|
-
|
72
|
-
before do
|
73
|
-
Sidekiq.error_handlers << error_handler
|
74
|
-
end
|
75
|
-
|
76
|
-
after do
|
77
|
-
Sidekiq.error_handlers.pop
|
78
|
-
end
|
79
|
-
|
80
|
-
it 'handles exceptions raised by the job' do
|
81
|
-
msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
|
82
|
-
begin
|
83
|
-
@processor.process(work(msg))
|
84
|
-
rescue TestException
|
85
|
-
end
|
86
|
-
assert_equal 1, errors.count
|
87
|
-
assert_instance_of TestException, errors.first
|
88
|
-
end
|
89
|
-
|
90
|
-
it 'handles exceptions raised by the reloader' do
|
91
|
-
msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['myarg'] })
|
92
|
-
@processor.instance_variable_set(:'@reloader', proc { raise TEST_EXCEPTION })
|
93
|
-
begin
|
94
|
-
@processor.process(work(msg))
|
95
|
-
rescue TestException
|
96
|
-
end
|
97
|
-
assert_equal 1, errors.count
|
98
|
-
assert_instance_of TestException, errors.first
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
describe 'acknowledgement' do
|
103
|
-
class ExceptionRaisingMiddleware
|
104
|
-
def initialize(raise_before_yield, raise_after_yield, skip)
|
105
|
-
@raise_before_yield = raise_before_yield
|
106
|
-
@raise_after_yield = raise_after_yield
|
107
|
-
@skip = skip
|
108
|
-
end
|
109
|
-
|
110
|
-
def call(worker, item, queue)
|
111
|
-
raise TEST_EXCEPTION if @raise_before_yield
|
112
|
-
yield unless @skip
|
113
|
-
raise TEST_EXCEPTION if @raise_after_yield
|
114
|
-
end
|
115
|
-
end
|
116
|
-
|
117
|
-
let(:raise_before_yield) { false }
|
118
|
-
let(:raise_after_yield) { false }
|
119
|
-
let(:skip_job) { false }
|
120
|
-
let(:worker_args) { ['myarg'] }
|
121
|
-
let(:work) { MiniTest::Mock.new }
|
122
|
-
|
123
|
-
before do
|
124
|
-
work.expect(:queue_name, 'queue:default')
|
125
|
-
work.expect(:job, Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => worker_args }))
|
126
|
-
Sidekiq.server_middleware do |chain|
|
127
|
-
chain.prepend ExceptionRaisingMiddleware, raise_before_yield, raise_after_yield, skip_job
|
128
|
-
end
|
129
|
-
end
|
130
|
-
|
131
|
-
after do
|
132
|
-
Sidekiq.server_middleware do |chain|
|
133
|
-
chain.remove ExceptionRaisingMiddleware
|
134
|
-
end
|
135
|
-
work.verify
|
136
|
-
end
|
137
|
-
|
138
|
-
describe 'middleware throws an exception before processing the work' do
|
139
|
-
let(:raise_before_yield) { true }
|
140
|
-
|
141
|
-
it 'does not ack' do
|
142
|
-
begin
|
143
|
-
@processor.process(work)
|
144
|
-
flunk "Expected #process to raise exception"
|
145
|
-
rescue TestException
|
146
|
-
end
|
147
|
-
end
|
148
|
-
end
|
149
|
-
|
150
|
-
describe 'middleware throws an exception after processing the work' do
|
151
|
-
let(:raise_after_yield) { true }
|
152
|
-
|
153
|
-
it 'acks the job' do
|
154
|
-
work.expect(:acknowledge, nil)
|
155
|
-
begin
|
156
|
-
@processor.process(work)
|
157
|
-
flunk "Expected #process to raise exception"
|
158
|
-
rescue TestException
|
159
|
-
end
|
160
|
-
end
|
161
|
-
end
|
162
|
-
|
163
|
-
describe 'middleware decides to skip work' do
|
164
|
-
let(:skip_job) { true }
|
165
|
-
|
166
|
-
it 'acks the job' do
|
167
|
-
work.expect(:acknowledge, nil)
|
168
|
-
@mgr.expect(:processor_done, nil, [@processor])
|
169
|
-
@processor.process(work)
|
170
|
-
end
|
171
|
-
end
|
172
|
-
|
173
|
-
describe 'worker raises an exception' do
|
174
|
-
let(:worker_args) { ['boom'] }
|
175
|
-
|
176
|
-
it 'acks the job' do
|
177
|
-
work.expect(:acknowledge, nil)
|
178
|
-
begin
|
179
|
-
@processor.process(work)
|
180
|
-
flunk "Expected #process to raise exception"
|
181
|
-
rescue TestException
|
182
|
-
end
|
183
|
-
end
|
184
|
-
end
|
185
|
-
|
186
|
-
describe 'everything goes well' do
|
187
|
-
it 'acks the job' do
|
188
|
-
work.expect(:acknowledge, nil)
|
189
|
-
@mgr.expect(:processor_done, nil, [@processor])
|
190
|
-
@processor.process(work)
|
191
|
-
end
|
192
|
-
end
|
193
|
-
end
|
194
|
-
|
195
|
-
describe 'stats' do
|
196
|
-
before do
|
197
|
-
Sidekiq.redis {|c| c.flushdb }
|
198
|
-
end
|
199
|
-
|
200
|
-
describe 'when successful' do
|
201
|
-
let(:processed_today_key) { "stat:processed:#{Time.now.utc.strftime("%Y-%m-%d")}" }
|
202
|
-
|
203
|
-
def successful_job
|
204
|
-
msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['myarg'] })
|
205
|
-
@mgr.expect(:processor_done, nil, [@processor])
|
206
|
-
@processor.process(work(msg))
|
207
|
-
end
|
208
|
-
|
209
|
-
it 'increments processed stat' do
|
210
|
-
Sidekiq::Processor::PROCESSED.value = 0
|
211
|
-
successful_job
|
212
|
-
assert_equal 1, Sidekiq::Processor::PROCESSED.value
|
213
|
-
end
|
214
|
-
end
|
215
|
-
|
216
|
-
describe 'when failed' do
|
217
|
-
let(:failed_today_key) { "stat:failed:#{Time.now.utc.strftime("%Y-%m-%d")}" }
|
218
|
-
|
219
|
-
def failed_job
|
220
|
-
msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
|
221
|
-
begin
|
222
|
-
@processor.process(work(msg))
|
223
|
-
rescue TestException
|
224
|
-
end
|
225
|
-
end
|
226
|
-
|
227
|
-
it 'increments failed stat' do
|
228
|
-
Sidekiq::Processor::FAILURE.value = 0
|
229
|
-
failed_job
|
230
|
-
assert_equal 1, Sidekiq::Processor::FAILURE.value
|
231
|
-
end
|
232
|
-
end
|
233
|
-
end
|
234
|
-
end
|
235
|
-
end
|
data/test/test_rails.rb
DELETED
@@ -1,22 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
require_relative 'helper'
|
3
|
-
|
4
|
-
$HAS_AJ = true
|
5
|
-
begin
|
6
|
-
require 'active_job'
|
7
|
-
rescue LoadError
|
8
|
-
$HAS_AJ = false
|
9
|
-
end
|
10
|
-
|
11
|
-
class TestRails < Sidekiq::Test
|
12
|
-
|
13
|
-
describe 'ActiveJob' do
|
14
|
-
it 'does not allow Sidekiq::Worker in AJ::Base classes' do
|
15
|
-
ex = assert_raises ArgumentError do
|
16
|
-
c = Class.new(ActiveJob::Base)
|
17
|
-
c.send(:include, Sidekiq::Worker)
|
18
|
-
end
|
19
|
-
assert_includes ex.message, "cannot include"
|
20
|
-
end if $HAS_AJ
|
21
|
-
end
|
22
|
-
end
|
@@ -1,132 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
require_relative 'helper'
|
3
|
-
|
4
|
-
class TestRedisConnection < Sidekiq::Test
|
5
|
-
|
6
|
-
describe ".create" do
|
7
|
-
|
8
|
-
it "creates a pooled redis connection" do
|
9
|
-
pool = Sidekiq::RedisConnection.create
|
10
|
-
assert_equal Redis, pool.checkout.class
|
11
|
-
end
|
12
|
-
|
13
|
-
describe "network_timeout" do
|
14
|
-
it "sets a custom network_timeout if specified" do
|
15
|
-
pool = Sidekiq::RedisConnection.create(:network_timeout => 8)
|
16
|
-
redis = pool.checkout
|
17
|
-
|
18
|
-
assert_equal 8, redis.client.timeout
|
19
|
-
end
|
20
|
-
|
21
|
-
it "uses the default network_timeout if none specified" do
|
22
|
-
pool = Sidekiq::RedisConnection.create
|
23
|
-
redis = pool.checkout
|
24
|
-
|
25
|
-
assert_equal 5, redis.client.timeout
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
describe "namespace" do
|
30
|
-
it "uses a given :namespace set by a symbol key" do
|
31
|
-
pool = Sidekiq::RedisConnection.create(:namespace => "xxx")
|
32
|
-
assert_equal "xxx", pool.checkout.namespace
|
33
|
-
end
|
34
|
-
|
35
|
-
it "uses a given :namespace set by a string key" do
|
36
|
-
pool = Sidekiq::RedisConnection.create("namespace" => "xxx")
|
37
|
-
assert_equal "xxx", pool.checkout.namespace
|
38
|
-
end
|
39
|
-
|
40
|
-
it "uses given :namespace over :namespace from Sidekiq.options" do
|
41
|
-
Sidekiq.options[:namespace] = "xxx"
|
42
|
-
pool = Sidekiq::RedisConnection.create(:namespace => "yyy")
|
43
|
-
assert_equal "yyy", pool.checkout.namespace
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
describe "socket path" do
|
48
|
-
it "uses a given :path" do
|
49
|
-
pool = Sidekiq::RedisConnection.create(:path => "/var/run/redis.sock")
|
50
|
-
assert_equal "unix", pool.checkout.client.scheme
|
51
|
-
assert_equal "redis:///var/run/redis.sock/0", pool.checkout.client.id
|
52
|
-
end
|
53
|
-
|
54
|
-
it "uses a given :path and :db" do
|
55
|
-
pool = Sidekiq::RedisConnection.create(:path => "/var/run/redis.sock", :db => 8)
|
56
|
-
assert_equal "unix", pool.checkout.client.scheme
|
57
|
-
assert_equal "redis:///var/run/redis.sock/8", pool.checkout.client.id
|
58
|
-
end
|
59
|
-
end
|
60
|
-
|
61
|
-
describe "pool_timeout" do
|
62
|
-
it "uses a given :timeout over the default of 1" do
|
63
|
-
pool = Sidekiq::RedisConnection.create(:pool_timeout => 5)
|
64
|
-
|
65
|
-
assert_equal 5, pool.instance_eval{ @timeout }
|
66
|
-
end
|
67
|
-
|
68
|
-
it "uses the default timeout of 1 if no override" do
|
69
|
-
pool = Sidekiq::RedisConnection.create
|
70
|
-
|
71
|
-
assert_equal 1, pool.instance_eval{ @timeout }
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
describe ".determine_redis_provider" do
|
77
|
-
|
78
|
-
before do
|
79
|
-
@old_env = ENV.to_hash
|
80
|
-
end
|
81
|
-
|
82
|
-
after do
|
83
|
-
ENV.update(@old_env)
|
84
|
-
end
|
85
|
-
|
86
|
-
def with_env_var(var, uri, skip_provider=false)
|
87
|
-
vars = ['REDISTOGO_URL', 'REDIS_PROVIDER', 'REDIS_URL'] - [var]
|
88
|
-
vars.each do |v|
|
89
|
-
next if skip_provider
|
90
|
-
ENV[v] = nil
|
91
|
-
end
|
92
|
-
ENV[var] = uri
|
93
|
-
assert_equal uri, Sidekiq::RedisConnection.__send__(:determine_redis_provider)
|
94
|
-
ENV[var] = nil
|
95
|
-
end
|
96
|
-
|
97
|
-
describe "with REDISTOGO_URL and a parallel REDIS_PROVIDER set" do
|
98
|
-
it "sets connection URI to the provider" do
|
99
|
-
uri = 'redis://sidekiq-redis-provider:6379/0'
|
100
|
-
provider = 'SIDEKIQ_REDIS_PROVIDER'
|
101
|
-
|
102
|
-
ENV['REDIS_PROVIDER'] = provider
|
103
|
-
ENV[provider] = uri
|
104
|
-
ENV['REDISTOGO_URL'] = 'redis://redis-to-go:6379/0'
|
105
|
-
with_env_var provider, uri, true
|
106
|
-
|
107
|
-
ENV[provider] = nil
|
108
|
-
end
|
109
|
-
end
|
110
|
-
|
111
|
-
describe "with REDIS_PROVIDER set" do
|
112
|
-
it "sets connection URI to the provider" do
|
113
|
-
uri = 'redis://sidekiq-redis-provider:6379/0'
|
114
|
-
provider = 'SIDEKIQ_REDIS_PROVIDER'
|
115
|
-
|
116
|
-
ENV['REDIS_PROVIDER'] = provider
|
117
|
-
ENV[provider] = uri
|
118
|
-
|
119
|
-
with_env_var provider, uri, true
|
120
|
-
|
121
|
-
ENV[provider] = nil
|
122
|
-
end
|
123
|
-
end
|
124
|
-
|
125
|
-
describe "with REDIS_URL set" do
|
126
|
-
it "sets connection URI to custom uri" do
|
127
|
-
with_env_var 'REDIS_URL', 'redis://redis-uri:6379/0'
|
128
|
-
end
|
129
|
-
end
|
130
|
-
|
131
|
-
end
|
132
|
-
end
|