sidekiq 4.2.4 → 6.4.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (143) hide show
  1. checksums.yaml +5 -5
  2. data/Changes.md +523 -0
  3. data/LICENSE +3 -3
  4. data/README.md +23 -36
  5. data/bin/sidekiq +26 -2
  6. data/bin/sidekiqload +28 -38
  7. data/bin/sidekiqmon +8 -0
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/job_spec.rb.erb +6 -0
  11. data/lib/generators/sidekiq/templates/job_test.rb.erb +8 -0
  12. data/lib/sidekiq/api.rb +403 -243
  13. data/lib/sidekiq/cli.rb +230 -211
  14. data/lib/sidekiq/client.rb +53 -64
  15. data/lib/sidekiq/delay.rb +43 -0
  16. data/lib/sidekiq/exception_handler.rb +12 -16
  17. data/lib/sidekiq/extensions/action_mailer.rb +15 -24
  18. data/lib/sidekiq/extensions/active_record.rb +15 -12
  19. data/lib/sidekiq/extensions/class_methods.rb +16 -13
  20. data/lib/sidekiq/extensions/generic_proxy.rb +14 -6
  21. data/lib/sidekiq/fetch.rb +39 -31
  22. data/lib/sidekiq/job.rb +13 -0
  23. data/lib/sidekiq/job_logger.rb +63 -0
  24. data/lib/sidekiq/job_retry.rb +261 -0
  25. data/lib/sidekiq/job_util.rb +65 -0
  26. data/lib/sidekiq/launcher.rb +170 -71
  27. data/lib/sidekiq/logger.rb +166 -0
  28. data/lib/sidekiq/manager.rb +21 -26
  29. data/lib/sidekiq/middleware/chain.rb +20 -8
  30. data/lib/sidekiq/middleware/current_attributes.rb +57 -0
  31. data/lib/sidekiq/middleware/i18n.rb +5 -7
  32. data/lib/sidekiq/monitor.rb +133 -0
  33. data/lib/sidekiq/paginator.rb +18 -14
  34. data/lib/sidekiq/processor.rb +161 -70
  35. data/lib/sidekiq/rails.rb +41 -73
  36. data/lib/sidekiq/redis_connection.rb +65 -20
  37. data/lib/sidekiq/scheduled.rb +95 -34
  38. data/lib/sidekiq/sd_notify.rb +149 -0
  39. data/lib/sidekiq/systemd.rb +24 -0
  40. data/lib/sidekiq/testing/inline.rb +2 -1
  41. data/lib/sidekiq/testing.rb +52 -26
  42. data/lib/sidekiq/util.rb +60 -14
  43. data/lib/sidekiq/version.rb +2 -1
  44. data/lib/sidekiq/web/action.rb +15 -15
  45. data/lib/sidekiq/web/application.rb +115 -89
  46. data/lib/sidekiq/web/csrf_protection.rb +180 -0
  47. data/lib/sidekiq/web/helpers.rb +151 -83
  48. data/lib/sidekiq/web/router.rb +27 -19
  49. data/lib/sidekiq/web.rb +65 -109
  50. data/lib/sidekiq/worker.rb +284 -41
  51. data/lib/sidekiq.rb +93 -60
  52. data/sidekiq.gemspec +24 -22
  53. data/web/assets/images/apple-touch-icon.png +0 -0
  54. data/web/assets/javascripts/application.js +83 -64
  55. data/web/assets/javascripts/dashboard.js +81 -85
  56. data/web/assets/stylesheets/application-dark.css +143 -0
  57. data/web/assets/stylesheets/application-rtl.css +242 -0
  58. data/web/assets/stylesheets/application.css +319 -143
  59. data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
  60. data/web/assets/stylesheets/bootstrap.css +2 -2
  61. data/web/locales/ar.yml +87 -0
  62. data/web/locales/de.yml +14 -2
  63. data/web/locales/en.yml +8 -1
  64. data/web/locales/es.yml +22 -5
  65. data/web/locales/fa.yml +80 -0
  66. data/web/locales/fr.yml +10 -3
  67. data/web/locales/he.yml +79 -0
  68. data/web/locales/ja.yml +12 -4
  69. data/web/locales/lt.yml +83 -0
  70. data/web/locales/pl.yml +4 -4
  71. data/web/locales/ru.yml +4 -0
  72. data/web/locales/ur.yml +80 -0
  73. data/web/locales/vi.yml +83 -0
  74. data/web/views/_footer.erb +5 -2
  75. data/web/views/_job_info.erb +4 -3
  76. data/web/views/_nav.erb +4 -18
  77. data/web/views/_paging.erb +1 -1
  78. data/web/views/_poll_link.erb +2 -5
  79. data/web/views/_summary.erb +7 -7
  80. data/web/views/busy.erb +60 -22
  81. data/web/views/dashboard.erb +23 -15
  82. data/web/views/dead.erb +3 -3
  83. data/web/views/layout.erb +14 -3
  84. data/web/views/morgue.erb +19 -12
  85. data/web/views/queue.erb +24 -14
  86. data/web/views/queues.erb +14 -4
  87. data/web/views/retries.erb +22 -13
  88. data/web/views/retry.erb +4 -4
  89. data/web/views/scheduled.erb +7 -4
  90. metadata +49 -198
  91. data/.github/contributing.md +0 -32
  92. data/.github/issue_template.md +0 -4
  93. data/.gitignore +0 -12
  94. data/.travis.yml +0 -12
  95. data/3.0-Upgrade.md +0 -70
  96. data/4.0-Upgrade.md +0 -53
  97. data/COMM-LICENSE +0 -95
  98. data/Ent-Changes.md +0 -146
  99. data/Gemfile +0 -29
  100. data/Pro-2.0-Upgrade.md +0 -138
  101. data/Pro-3.0-Upgrade.md +0 -44
  102. data/Pro-Changes.md +0 -585
  103. data/Rakefile +0 -9
  104. data/bin/sidekiqctl +0 -99
  105. data/code_of_conduct.md +0 -50
  106. data/lib/generators/sidekiq/templates/worker_spec.rb.erb +0 -6
  107. data/lib/generators/sidekiq/templates/worker_test.rb.erb +0 -8
  108. data/lib/generators/sidekiq/worker_generator.rb +0 -49
  109. data/lib/sidekiq/core_ext.rb +0 -106
  110. data/lib/sidekiq/logging.rb +0 -106
  111. data/lib/sidekiq/middleware/server/active_record.rb +0 -13
  112. data/lib/sidekiq/middleware/server/logging.rb +0 -40
  113. data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -205
  114. data/test/config.yml +0 -9
  115. data/test/env_based_config.yml +0 -11
  116. data/test/fake_env.rb +0 -1
  117. data/test/fixtures/en.yml +0 -2
  118. data/test/helper.rb +0 -75
  119. data/test/test_actors.rb +0 -138
  120. data/test/test_api.rb +0 -528
  121. data/test/test_cli.rb +0 -418
  122. data/test/test_client.rb +0 -266
  123. data/test/test_exception_handler.rb +0 -56
  124. data/test/test_extensions.rb +0 -127
  125. data/test/test_fetch.rb +0 -50
  126. data/test/test_launcher.rb +0 -95
  127. data/test/test_logging.rb +0 -35
  128. data/test/test_manager.rb +0 -50
  129. data/test/test_middleware.rb +0 -158
  130. data/test/test_processor.rb +0 -235
  131. data/test/test_rails.rb +0 -22
  132. data/test/test_redis_connection.rb +0 -132
  133. data/test/test_retry.rb +0 -326
  134. data/test/test_retry_exhausted.rb +0 -149
  135. data/test/test_scheduled.rb +0 -115
  136. data/test/test_scheduling.rb +0 -58
  137. data/test/test_sidekiq.rb +0 -107
  138. data/test/test_testing.rb +0 -143
  139. data/test/test_testing_fake.rb +0 -357
  140. data/test/test_testing_inline.rb +0 -94
  141. data/test/test_util.rb +0 -13
  142. data/test/test_web.rb +0 -726
  143. data/test/test_web_helpers.rb +0 -54
@@ -1,235 +0,0 @@
1
- # frozen_string_literal: true
2
- require_relative 'helper'
3
- require 'sidekiq/fetch'
4
- require 'sidekiq/cli'
5
- require 'sidekiq/processor'
6
-
7
- class TestProcessor < Sidekiq::Test
8
- TestException = Class.new(StandardError)
9
- TEST_EXCEPTION = TestException.new("kerboom!")
10
-
11
- describe 'processor' do
12
- before do
13
- $invokes = 0
14
- @mgr = Minitest::Mock.new
15
- @mgr.expect(:options, {:queues => ['default']})
16
- @mgr.expect(:options, {:queues => ['default']})
17
- @processor = ::Sidekiq::Processor.new(@mgr)
18
- end
19
-
20
- class MockWorker
21
- include Sidekiq::Worker
22
- def perform(args)
23
- raise TEST_EXCEPTION if args == 'boom'
24
- args.pop if args.is_a? Array
25
- $invokes += 1
26
- end
27
- end
28
-
29
- def work(msg, queue='queue:default')
30
- Sidekiq::BasicFetch::UnitOfWork.new(queue, msg)
31
- end
32
-
33
- it 'processes as expected' do
34
- msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['myarg'] })
35
- @processor.process(work(msg))
36
- assert_equal 1, $invokes
37
- end
38
-
39
- it 'executes a worker as expected' do
40
- worker = Minitest::Mock.new
41
- worker.expect(:perform, nil, [1, 2, 3])
42
- @processor.execute_job(worker, [1, 2, 3])
43
- end
44
-
45
- it 're-raises exceptions after handling' do
46
- msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
47
- re_raise = false
48
-
49
- begin
50
- @processor.process(work(msg))
51
- flunk "Expected exception"
52
- rescue TestException
53
- re_raise = true
54
- end
55
-
56
- assert_equal 0, $invokes
57
- assert re_raise, "does not re-raise exceptions after handling"
58
- end
59
-
60
- it 'does not modify original arguments' do
61
- msg = { 'class' => MockWorker.to_s, 'args' => [['myarg']] }
62
- msgstr = Sidekiq.dump_json(msg)
63
- @mgr.expect(:processor_done, nil, [@processor])
64
- @processor.process(work(msgstr))
65
- assert_equal [['myarg']], msg['args']
66
- end
67
-
68
- describe 'exception handling' do
69
- let(:errors) { [] }
70
- let(:error_handler) { proc { |ex, _| errors << ex } }
71
-
72
- before do
73
- Sidekiq.error_handlers << error_handler
74
- end
75
-
76
- after do
77
- Sidekiq.error_handlers.pop
78
- end
79
-
80
- it 'handles exceptions raised by the job' do
81
- msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
82
- begin
83
- @processor.process(work(msg))
84
- rescue TestException
85
- end
86
- assert_equal 1, errors.count
87
- assert_instance_of TestException, errors.first
88
- end
89
-
90
- it 'handles exceptions raised by the reloader' do
91
- msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['myarg'] })
92
- @processor.instance_variable_set(:'@reloader', proc { raise TEST_EXCEPTION })
93
- begin
94
- @processor.process(work(msg))
95
- rescue TestException
96
- end
97
- assert_equal 1, errors.count
98
- assert_instance_of TestException, errors.first
99
- end
100
- end
101
-
102
- describe 'acknowledgement' do
103
- class ExceptionRaisingMiddleware
104
- def initialize(raise_before_yield, raise_after_yield, skip)
105
- @raise_before_yield = raise_before_yield
106
- @raise_after_yield = raise_after_yield
107
- @skip = skip
108
- end
109
-
110
- def call(worker, item, queue)
111
- raise TEST_EXCEPTION if @raise_before_yield
112
- yield unless @skip
113
- raise TEST_EXCEPTION if @raise_after_yield
114
- end
115
- end
116
-
117
- let(:raise_before_yield) { false }
118
- let(:raise_after_yield) { false }
119
- let(:skip_job) { false }
120
- let(:worker_args) { ['myarg'] }
121
- let(:work) { MiniTest::Mock.new }
122
-
123
- before do
124
- work.expect(:queue_name, 'queue:default')
125
- work.expect(:job, Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => worker_args }))
126
- Sidekiq.server_middleware do |chain|
127
- chain.prepend ExceptionRaisingMiddleware, raise_before_yield, raise_after_yield, skip_job
128
- end
129
- end
130
-
131
- after do
132
- Sidekiq.server_middleware do |chain|
133
- chain.remove ExceptionRaisingMiddleware
134
- end
135
- work.verify
136
- end
137
-
138
- describe 'middleware throws an exception before processing the work' do
139
- let(:raise_before_yield) { true }
140
-
141
- it 'does not ack' do
142
- begin
143
- @processor.process(work)
144
- flunk "Expected #process to raise exception"
145
- rescue TestException
146
- end
147
- end
148
- end
149
-
150
- describe 'middleware throws an exception after processing the work' do
151
- let(:raise_after_yield) { true }
152
-
153
- it 'acks the job' do
154
- work.expect(:acknowledge, nil)
155
- begin
156
- @processor.process(work)
157
- flunk "Expected #process to raise exception"
158
- rescue TestException
159
- end
160
- end
161
- end
162
-
163
- describe 'middleware decides to skip work' do
164
- let(:skip_job) { true }
165
-
166
- it 'acks the job' do
167
- work.expect(:acknowledge, nil)
168
- @mgr.expect(:processor_done, nil, [@processor])
169
- @processor.process(work)
170
- end
171
- end
172
-
173
- describe 'worker raises an exception' do
174
- let(:worker_args) { ['boom'] }
175
-
176
- it 'acks the job' do
177
- work.expect(:acknowledge, nil)
178
- begin
179
- @processor.process(work)
180
- flunk "Expected #process to raise exception"
181
- rescue TestException
182
- end
183
- end
184
- end
185
-
186
- describe 'everything goes well' do
187
- it 'acks the job' do
188
- work.expect(:acknowledge, nil)
189
- @mgr.expect(:processor_done, nil, [@processor])
190
- @processor.process(work)
191
- end
192
- end
193
- end
194
-
195
- describe 'stats' do
196
- before do
197
- Sidekiq.redis {|c| c.flushdb }
198
- end
199
-
200
- describe 'when successful' do
201
- let(:processed_today_key) { "stat:processed:#{Time.now.utc.strftime("%Y-%m-%d")}" }
202
-
203
- def successful_job
204
- msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['myarg'] })
205
- @mgr.expect(:processor_done, nil, [@processor])
206
- @processor.process(work(msg))
207
- end
208
-
209
- it 'increments processed stat' do
210
- Sidekiq::Processor::PROCESSED.value = 0
211
- successful_job
212
- assert_equal 1, Sidekiq::Processor::PROCESSED.value
213
- end
214
- end
215
-
216
- describe 'when failed' do
217
- let(:failed_today_key) { "stat:failed:#{Time.now.utc.strftime("%Y-%m-%d")}" }
218
-
219
- def failed_job
220
- msg = Sidekiq.dump_json({ 'class' => MockWorker.to_s, 'args' => ['boom'] })
221
- begin
222
- @processor.process(work(msg))
223
- rescue TestException
224
- end
225
- end
226
-
227
- it 'increments failed stat' do
228
- Sidekiq::Processor::FAILURE.value = 0
229
- failed_job
230
- assert_equal 1, Sidekiq::Processor::FAILURE.value
231
- end
232
- end
233
- end
234
- end
235
- end
data/test/test_rails.rb DELETED
@@ -1,22 +0,0 @@
1
- # frozen_string_literal: true
2
- require_relative 'helper'
3
-
4
- $HAS_AJ = true
5
- begin
6
- require 'active_job'
7
- rescue LoadError
8
- $HAS_AJ = false
9
- end
10
-
11
- class TestRails < Sidekiq::Test
12
-
13
- describe 'ActiveJob' do
14
- it 'does not allow Sidekiq::Worker in AJ::Base classes' do
15
- ex = assert_raises ArgumentError do
16
- c = Class.new(ActiveJob::Base)
17
- c.send(:include, Sidekiq::Worker)
18
- end
19
- assert_includes ex.message, "cannot include"
20
- end if $HAS_AJ
21
- end
22
- end
@@ -1,132 +0,0 @@
1
- # frozen_string_literal: true
2
- require_relative 'helper'
3
-
4
- class TestRedisConnection < Sidekiq::Test
5
-
6
- describe ".create" do
7
-
8
- it "creates a pooled redis connection" do
9
- pool = Sidekiq::RedisConnection.create
10
- assert_equal Redis, pool.checkout.class
11
- end
12
-
13
- describe "network_timeout" do
14
- it "sets a custom network_timeout if specified" do
15
- pool = Sidekiq::RedisConnection.create(:network_timeout => 8)
16
- redis = pool.checkout
17
-
18
- assert_equal 8, redis.client.timeout
19
- end
20
-
21
- it "uses the default network_timeout if none specified" do
22
- pool = Sidekiq::RedisConnection.create
23
- redis = pool.checkout
24
-
25
- assert_equal 5, redis.client.timeout
26
- end
27
- end
28
-
29
- describe "namespace" do
30
- it "uses a given :namespace set by a symbol key" do
31
- pool = Sidekiq::RedisConnection.create(:namespace => "xxx")
32
- assert_equal "xxx", pool.checkout.namespace
33
- end
34
-
35
- it "uses a given :namespace set by a string key" do
36
- pool = Sidekiq::RedisConnection.create("namespace" => "xxx")
37
- assert_equal "xxx", pool.checkout.namespace
38
- end
39
-
40
- it "uses given :namespace over :namespace from Sidekiq.options" do
41
- Sidekiq.options[:namespace] = "xxx"
42
- pool = Sidekiq::RedisConnection.create(:namespace => "yyy")
43
- assert_equal "yyy", pool.checkout.namespace
44
- end
45
- end
46
-
47
- describe "socket path" do
48
- it "uses a given :path" do
49
- pool = Sidekiq::RedisConnection.create(:path => "/var/run/redis.sock")
50
- assert_equal "unix", pool.checkout.client.scheme
51
- assert_equal "redis:///var/run/redis.sock/0", pool.checkout.client.id
52
- end
53
-
54
- it "uses a given :path and :db" do
55
- pool = Sidekiq::RedisConnection.create(:path => "/var/run/redis.sock", :db => 8)
56
- assert_equal "unix", pool.checkout.client.scheme
57
- assert_equal "redis:///var/run/redis.sock/8", pool.checkout.client.id
58
- end
59
- end
60
-
61
- describe "pool_timeout" do
62
- it "uses a given :timeout over the default of 1" do
63
- pool = Sidekiq::RedisConnection.create(:pool_timeout => 5)
64
-
65
- assert_equal 5, pool.instance_eval{ @timeout }
66
- end
67
-
68
- it "uses the default timeout of 1 if no override" do
69
- pool = Sidekiq::RedisConnection.create
70
-
71
- assert_equal 1, pool.instance_eval{ @timeout }
72
- end
73
- end
74
- end
75
-
76
- describe ".determine_redis_provider" do
77
-
78
- before do
79
- @old_env = ENV.to_hash
80
- end
81
-
82
- after do
83
- ENV.update(@old_env)
84
- end
85
-
86
- def with_env_var(var, uri, skip_provider=false)
87
- vars = ['REDISTOGO_URL', 'REDIS_PROVIDER', 'REDIS_URL'] - [var]
88
- vars.each do |v|
89
- next if skip_provider
90
- ENV[v] = nil
91
- end
92
- ENV[var] = uri
93
- assert_equal uri, Sidekiq::RedisConnection.__send__(:determine_redis_provider)
94
- ENV[var] = nil
95
- end
96
-
97
- describe "with REDISTOGO_URL and a parallel REDIS_PROVIDER set" do
98
- it "sets connection URI to the provider" do
99
- uri = 'redis://sidekiq-redis-provider:6379/0'
100
- provider = 'SIDEKIQ_REDIS_PROVIDER'
101
-
102
- ENV['REDIS_PROVIDER'] = provider
103
- ENV[provider] = uri
104
- ENV['REDISTOGO_URL'] = 'redis://redis-to-go:6379/0'
105
- with_env_var provider, uri, true
106
-
107
- ENV[provider] = nil
108
- end
109
- end
110
-
111
- describe "with REDIS_PROVIDER set" do
112
- it "sets connection URI to the provider" do
113
- uri = 'redis://sidekiq-redis-provider:6379/0'
114
- provider = 'SIDEKIQ_REDIS_PROVIDER'
115
-
116
- ENV['REDIS_PROVIDER'] = provider
117
- ENV[provider] = uri
118
-
119
- with_env_var provider, uri, true
120
-
121
- ENV[provider] = nil
122
- end
123
- end
124
-
125
- describe "with REDIS_URL set" do
126
- it "sets connection URI to custom uri" do
127
- with_env_var 'REDIS_URL', 'redis://redis-uri:6379/0'
128
- end
129
- end
130
-
131
- end
132
- end