sidekiq 6.1.1 → 6.5.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (118) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +250 -3
  3. data/LICENSE +3 -3
  4. data/README.md +10 -6
  5. data/bin/sidekiq +3 -3
  6. data/bin/sidekiqload +70 -66
  7. data/bin/sidekiqmon +1 -1
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  11. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  12. data/lib/sidekiq/api.rb +352 -156
  13. data/lib/sidekiq/cli.rb +86 -41
  14. data/lib/sidekiq/client.rb +49 -73
  15. data/lib/sidekiq/{util.rb → component.rb} +12 -14
  16. data/lib/sidekiq/delay.rb +3 -1
  17. data/lib/sidekiq/extensions/action_mailer.rb +3 -2
  18. data/lib/sidekiq/extensions/active_record.rb +1 -1
  19. data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
  20. data/lib/sidekiq/fetch.rb +31 -20
  21. data/lib/sidekiq/job.rb +13 -0
  22. data/lib/sidekiq/job_logger.rb +16 -28
  23. data/lib/sidekiq/job_retry.rb +79 -59
  24. data/lib/sidekiq/job_util.rb +71 -0
  25. data/lib/sidekiq/launcher.rb +126 -65
  26. data/lib/sidekiq/logger.rb +11 -20
  27. data/lib/sidekiq/manager.rb +35 -34
  28. data/lib/sidekiq/metrics/deploy.rb +47 -0
  29. data/lib/sidekiq/metrics/query.rb +153 -0
  30. data/lib/sidekiq/metrics/shared.rb +94 -0
  31. data/lib/sidekiq/metrics/tracking.rb +134 -0
  32. data/lib/sidekiq/middleware/chain.rb +88 -42
  33. data/lib/sidekiq/middleware/current_attributes.rb +63 -0
  34. data/lib/sidekiq/middleware/i18n.rb +6 -4
  35. data/lib/sidekiq/middleware/modules.rb +21 -0
  36. data/lib/sidekiq/monitor.rb +2 -2
  37. data/lib/sidekiq/paginator.rb +17 -9
  38. data/lib/sidekiq/processor.rb +47 -41
  39. data/lib/sidekiq/rails.rb +32 -4
  40. data/lib/sidekiq/redis_client_adapter.rb +154 -0
  41. data/lib/sidekiq/redis_connection.rb +84 -55
  42. data/lib/sidekiq/ring_buffer.rb +29 -0
  43. data/lib/sidekiq/scheduled.rb +96 -32
  44. data/lib/sidekiq/testing/inline.rb +4 -4
  45. data/lib/sidekiq/testing.rb +38 -39
  46. data/lib/sidekiq/transaction_aware_client.rb +45 -0
  47. data/lib/sidekiq/version.rb +1 -1
  48. data/lib/sidekiq/web/action.rb +3 -3
  49. data/lib/sidekiq/web/application.rb +41 -16
  50. data/lib/sidekiq/web/csrf_protection.rb +32 -5
  51. data/lib/sidekiq/web/helpers.rb +52 -30
  52. data/lib/sidekiq/web/router.rb +4 -1
  53. data/lib/sidekiq/web.rb +38 -78
  54. data/lib/sidekiq/worker.rb +142 -16
  55. data/lib/sidekiq.rb +114 -31
  56. data/sidekiq.gemspec +12 -4
  57. data/web/assets/images/apple-touch-icon.png +0 -0
  58. data/web/assets/javascripts/application.js +114 -60
  59. data/web/assets/javascripts/chart.min.js +13 -0
  60. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  61. data/web/assets/javascripts/dashboard.js +50 -67
  62. data/web/assets/javascripts/graph.js +16 -0
  63. data/web/assets/javascripts/metrics.js +262 -0
  64. data/web/assets/stylesheets/application-dark.css +61 -51
  65. data/web/assets/stylesheets/application-rtl.css +0 -4
  66. data/web/assets/stylesheets/application.css +84 -243
  67. data/web/locales/ar.yml +8 -2
  68. data/web/locales/el.yml +43 -19
  69. data/web/locales/en.yml +11 -1
  70. data/web/locales/es.yml +18 -2
  71. data/web/locales/fr.yml +8 -1
  72. data/web/locales/ja.yml +10 -0
  73. data/web/locales/lt.yml +1 -1
  74. data/web/locales/pt-br.yml +27 -9
  75. data/web/locales/ru.yml +4 -0
  76. data/web/locales/zh-cn.yml +36 -11
  77. data/web/locales/zh-tw.yml +32 -7
  78. data/web/views/_footer.erb +1 -1
  79. data/web/views/_job_info.erb +1 -1
  80. data/web/views/_nav.erb +1 -1
  81. data/web/views/_poll_link.erb +2 -5
  82. data/web/views/_summary.erb +7 -7
  83. data/web/views/busy.erb +57 -21
  84. data/web/views/dashboard.erb +23 -14
  85. data/web/views/dead.erb +1 -1
  86. data/web/views/layout.erb +2 -1
  87. data/web/views/metrics.erb +69 -0
  88. data/web/views/metrics_for_job.erb +87 -0
  89. data/web/views/morgue.erb +6 -6
  90. data/web/views/queue.erb +15 -11
  91. data/web/views/queues.erb +4 -4
  92. data/web/views/retries.erb +7 -7
  93. data/web/views/retry.erb +1 -1
  94. data/web/views/scheduled.erb +1 -1
  95. metadata +52 -39
  96. data/.circleci/config.yml +0 -71
  97. data/.github/contributing.md +0 -32
  98. data/.github/issue_template.md +0 -11
  99. data/.gitignore +0 -13
  100. data/.standard.yml +0 -20
  101. data/3.0-Upgrade.md +0 -70
  102. data/4.0-Upgrade.md +0 -53
  103. data/5.0-Upgrade.md +0 -56
  104. data/6.0-Upgrade.md +0 -72
  105. data/COMM-LICENSE +0 -97
  106. data/Ent-2.0-Upgrade.md +0 -37
  107. data/Ent-Changes.md +0 -275
  108. data/Gemfile +0 -24
  109. data/Gemfile.lock +0 -208
  110. data/Pro-2.0-Upgrade.md +0 -138
  111. data/Pro-3.0-Upgrade.md +0 -44
  112. data/Pro-4.0-Upgrade.md +0 -35
  113. data/Pro-5.0-Upgrade.md +0 -25
  114. data/Pro-Changes.md +0 -795
  115. data/Rakefile +0 -10
  116. data/code_of_conduct.md +0 -50
  117. data/lib/generators/sidekiq/worker_generator.rb +0 -57
  118. data/lib/sidekiq/exception_handler.rb +0 -27
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3e65c309b08d7da496e7181ff91980d43d8cf56e226fcd921c00e392dbecc22f
4
- data.tar.gz: 45ff998fd67a0ca37c99176bc1a92c4160569106ff406d75c37e2a5968501d25
3
+ metadata.gz: b12bc33857058ca07b776ff3f9e39eb98a84533c8f1c78367626a6a514d53607
4
+ data.tar.gz: d783f3060fc348786c7128ffc3b14703d9ee676a1a4972d88bc2e995fbabfd16
5
5
  SHA512:
6
- metadata.gz: fd592b7657c04d8ff7363157a965233a0e6a74889d7ae44a9bf398143ab7847776dbb52cd5e1328125afabe25be79f6c72ae9f03283672e49ae3868bc99037ec
7
- data.tar.gz: 10b3e967c94933d8a8b269499a71d9690759504398855edde5197ef6f6648d26fa201e131d9c7650c64b6baa2b5306cd40fe618c50e122d11247d8cd2e7a8bbb
6
+ metadata.gz: 0471f10240cbf2a1e2337e50111690a32153da3fa23f3e8462ff545dcc9cde06d67574cc26ed1c2be3fed88d31a34002ec30641a6c40a4f668a44005e9d5d297
7
+ data.tar.gz: 88f36a02144dbdf6fa58eb5d58f9615d99321049a76717dd2a4bd03a0bc7ebfe69e4709b860afd22782c80e5a868d8c85026d15ce42824675e3f6f09534c6335
data/Changes.md CHANGED
@@ -1,8 +1,248 @@
1
1
  # Sidekiq Changes
2
2
 
3
- [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
3
+ [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
4
4
 
5
- HEAD
5
+ 6.5.10
6
+ ----------
7
+
8
+ - Web UI DoS vector [#6045] CVE-2023-26141
9
+ - Fix broadcast logger with Rails 7.1 [#6054]
10
+
11
+ 6.5.9
12
+ ----------
13
+
14
+ - Ensure Sidekiq.options[:environment] == RAILS_ENV [#5932]
15
+
16
+ 6.5.8
17
+ ----------
18
+
19
+ - Fail if using a bad version of scout_apm [#5616]
20
+ - Add pagination to Busy page [#5556]
21
+ - Speed up WorkSet#each [#5559]
22
+ - Adjust CurrentAttributes to work with the String class name so we aren't referencing
23
+ the Class within a Rails initializer [#5536]
24
+
25
+ 6.5.7
26
+ ----------
27
+
28
+ - Updates for JA and ZH locales
29
+ - Further optimizations for scheduled polling [#5513]
30
+
31
+ 6.5.6
32
+ ----------
33
+
34
+ - Fix deprecation warnings with redis-rb 4.8.0 [#5484]
35
+ - Lock redis-rb to < 5.0 as we are moving to redis-client in Sidekiq 7.0
36
+
37
+ 6.5.5
38
+ ----------
39
+
40
+ - Fix require issue with job_retry.rb [#5462]
41
+ - Improve Sidekiq::Web compatibility with Rack 3.x
42
+
43
+ 6.5.4
44
+ ----------
45
+
46
+ - Fix invalid code on Ruby 2.5 [#5460]
47
+ - Fix further metrics dependency issues [#5457]
48
+
49
+ 6.5.3
50
+ ----------
51
+
52
+ - Don't require metrics code without explicit opt-in [#5456]
53
+
54
+ 6.5.2
55
+ ----------
56
+
57
+ - [Job Metrics are under active development, help wanted!](https://github.com/mperham/sidekiq/wiki/Metrics#contributing) **BETA**
58
+ - Add `Context` column on queue page which shows any CurrentAttributes [#5450]
59
+ - `sidekiq_retry_in` may now return `:discard` or `:kill` to dynamically stop job retries [#5406]
60
+ - Smarter sorting of processes in /busy Web UI [#5398]
61
+ - Fix broken hamburger menu in mobile UI [#5428]
62
+ - Require redis-rb 4.5.0. Note that Sidekiq will break if you use the
63
+ [`Redis.exists_returns_integer = false`](https://github.com/redis/redis-rb/blob/master/CHANGELOG.md#450) flag. [#5394]
64
+
65
+ 6.5.1
66
+ ----------
67
+
68
+ - Fix `push_bulk` breakage [#5387]
69
+
70
+ 6.5.0
71
+ ---------
72
+
73
+ - Substantial refactoring of Sidekiq server internals, part of a larger effort
74
+ to reduce Sidekiq's internal usage of global methods and data, see [docs/global_to_local.md](docs/global_to_local.md) and [docs/middleware.md](docs/middleware.md).
75
+ - **Add beta support for the `redis-client` gem**. This will become the default Redis driver in Sidekiq 7.0. [#5298]
76
+ Read more: https://github.com/mperham/sidekiq/wiki/Using-redis-client
77
+ - **Add beta support for DB transaction-aware client** [#5291]
78
+ Add this line to your initializer and any jobs created during a transaction
79
+ will only be pushed to Redis **after the transaction commits**. You will need to add the
80
+ `after_commit_everywhere` gem to your Gemfile.
81
+ ```ruby
82
+ Sidekiq.transactional_push!
83
+ ```
84
+ This feature does not have a lot of production usage yet; please try it out and let us
85
+ know if you have any issues. It will be fully supported in Sidekiq 7.0 or removed if it
86
+ proves problematic.
87
+ - Fix regression with middleware arguments [#5312]
88
+
89
+ 6.4.2
90
+ ---------
91
+
92
+ - Strict argument checking now runs after client-side middleware [#5246]
93
+ - Fix page events with live polling [#5184]
94
+ - Many under-the-hood changes to remove all usage of the term "worker"
95
+ from the Sidekiq codebase and APIs. This mostly involved RDoc and local
96
+ variable names but a few constants and public APIs were changed. The old
97
+ APIs will be removed in Sidekiq 7.0.
98
+ ```
99
+ Sidekiq::DEFAULT_WORKER_OPTIONS -> Sidekiq.default_job_options
100
+ Sidekiq.default_worker_options -> Sidekiq.default_job_options
101
+ Sidekiq::Queues["default"].jobs_by_worker(HardJob) -> Sidekiq::Queues["default"].jobs_by_class(HardJob)
102
+ ```
103
+
104
+ 6.4.1
105
+ ---------
106
+
107
+ - Fix pipeline/multi deprecations in redis-rb 4.6
108
+ - Fix sidekiq.yml YAML load errors on Ruby 3.1 [#5141]
109
+ - Sharding support for `perform_bulk` [#5129]
110
+ - Refactor job logger for SPEEEEEEED
111
+
112
+ 6.4.0
113
+ ---------
114
+
115
+ - **SECURITY**: Validate input to avoid possible DoS in Web UI.
116
+ - Add **strict argument checking** [#5071]
117
+ Sidekiq will now log a warning if JSON-unsafe arguments are passed to `perform_async`.
118
+ Add `Sidekiq.strict_args!(false)` to your initializer to disable this warning.
119
+ This warning will switch to an exception in Sidekiq 7.0.
120
+ - Note that Delayed Extensions will be removed in Sidekiq 7.0 [#5076]
121
+ - Add `perform_{inline,sync}` in Sidekiq::Job to run a job synchronously [#5061, hasan-ally]
122
+ ```ruby
123
+ SomeJob.perform_async(args...)
124
+ SomeJob.perform_sync(args...)
125
+ SomeJob.perform_inline(args...)
126
+ ```
127
+ You can also dynamically redirect a job to run synchronously:
128
+ ```ruby
129
+ SomeJob.set("sync": true).perform_async(args...) # will run via perform_inline
130
+ ```
131
+ - Replace Sidekiq::Worker `app/workers` generator with Sidekiq::Job `app/sidekiq` generator [#5055]
132
+ ```
133
+ bin/rails generate sidekiq:job ProcessOrderJob
134
+ ```
135
+ - Fix job retries losing CurrentAttributes [#5090]
136
+ - Tweak shutdown to give long-running threads time to cleanup [#5095]
137
+
138
+ 6.3.1
139
+ ---------
140
+
141
+ - Fix keyword arguments error with CurrentAttributes on Ruby 3.0 [#5048]
142
+
143
+ 6.3.0
144
+ ---------
145
+
146
+ - **BREAK**: The Web UI has been refactored to remove jQuery. Any UI extensions
147
+ which use jQuery will break.
148
+ - **FEATURE**: Sidekiq.logger has been enhanced so any `Rails.logger`
149
+ output in jobs now shows up in the Sidekiq console. Remove any logger
150
+ hacks in your initializer and see if it Just Works™ now. [#5021]
151
+ - **FEATURE**: Add `Sidekiq::Job` alias for `Sidekiq::Worker`, to better
152
+ reflect industry standard terminology. You can now do this:
153
+ ```ruby
154
+ class MyJob
155
+ include Sidekiq::Job
156
+ sidekiq_options ...
157
+ def perform(args)
158
+ end
159
+ end
160
+ ```
161
+ - **FEATURE**: Support for serializing ActiveSupport::CurrentAttributes into each job. [#4982]
162
+ ```ruby
163
+ # config/initializers/sidekiq.rb
164
+ require "sidekiq/middleware/current_attributes"
165
+ Sidekiq::CurrentAttributes.persist(Myapp::Current) # Your AS::CurrentAttributes singleton
166
+ ```
167
+ - **FEATURE**: Add `Sidekiq::Worker.perform_bulk` for enqueuing jobs in bulk,
168
+ similar to `Sidekiq::Client.push_bulk` [#5042]
169
+ ```ruby
170
+ MyJob.perform_bulk([[1], [2], [3]])
171
+ ```
172
+ - Implement `queue_as`, `wait` and `wait_until` for ActiveJob compatibility [#5003]
173
+ - Scheduler now uses Lua to reduce Redis load and network roundtrips [#5044]
174
+ - Retry Redis operation if we get an `UNBLOCKED` Redis error [#4985]
175
+ - Run existing signal traps, if any, before running Sidekiq's trap [#4991]
176
+ - Fix fetch bug when using weighted queues which caused Sidekiq to stop
177
+ processing queues randomly [#5031]
178
+
179
+ 6.2.2
180
+ ---------
181
+
182
+ - Reduce retry jitter, add jitter to `sidekiq_retry_in` values [#4957]
183
+ - Minimize scheduler load on Redis at scale [#4882]
184
+ - Improve logging of delay jobs [#4904, BuonOno]
185
+ - Minor CSS improvements for buttons and tables, design PRs always welcome!
186
+ - Tweak Web UI `Cache-Control` header [#4966]
187
+ - Rename internal API class `Sidekiq::Job` to `Sidekiq::JobRecord` [#4955]
188
+
189
+ 6.2.1
190
+ ---------
191
+
192
+ - Update RTT warning logic to handle transient RTT spikes [#4851]
193
+ - Fix very low priority CVE on unescaped queue name [#4852]
194
+ - Add note about sessions and Rails apps in API mode
195
+
196
+ 6.2.0
197
+ ---------
198
+
199
+ - Store Redis RTT and log if poor [#4824]
200
+ - Add process/thread stats to Busy page [#4806]
201
+ - Improve Web UI on mobile devices [#4840]
202
+ - **Refactor Web UI session usage** [#4804]
203
+ Numerous people have hit "Forbidden" errors and struggled with Sidekiq's
204
+ Web UI session requirement. If you have code in your initializer for
205
+ Web sessions, it's quite possible it will need to be removed. Here's
206
+ an overview:
207
+ ```
208
+ Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app,
209
+ make sure you mount Sidekiq::Web *inside* your routes in `config/routes.rb` so
210
+ Sidekiq can reuse the Rails session:
211
+
212
+ Rails.application.routes.draw do
213
+ mount Sidekiq::Web => "/sidekiq"
214
+ ....
215
+ end
216
+
217
+ If this is a bare Rack app, use a session middleware before Sidekiq::Web:
218
+
219
+ # first, use IRB to create a shared secret key for sessions and commit it
220
+ require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
221
+
222
+ # now, update your Rack app to include the secret with a session cookie middleware
223
+ use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
224
+ run Sidekiq::Web
225
+
226
+ If this is a Rails app in API mode, you need to enable sessions.
227
+
228
+ https://guides.rubyonrails.org/api_app.html#using-session-middlewares
229
+ ```
230
+
231
+ 6.1.3
232
+ ---------
233
+
234
+ - Warn if Redis is configured to evict data under memory pressure [#4752]
235
+ - Add process RSS on the Busy page [#4717]
236
+
237
+ 6.1.2
238
+ ---------
239
+
240
+ - Improve readability in dark mode Web UI [#4674]
241
+ - Fix Web UI crash with corrupt session [#4672]
242
+ - Allow middleware to yield arguments [#4673, @eugeneius]
243
+ - Migrate CI from CircleCI to GitHub Actions [#4677]
244
+
245
+ 6.1.1
6
246
  ---------
7
247
 
8
248
  - Jobs are now sorted by age in the Busy Workers table. [#4641]
@@ -142,7 +382,7 @@ assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs_for(FooMailer).size
142
382
 
143
383
  This release has major breaking changes. Read and test carefully in production.
144
384
 
145
- - With Rails 6.0.1+, ActiveJobs can now use `sidekiq_options` directly to configure Sidekiq
385
+ - With Rails 6.0.2+, ActiveJobs can now use `sidekiq_options` directly to configure Sidekiq
146
386
  features/internals like the retry subsystem. [#4213, pirj]
147
387
  ```ruby
148
388
  class MyJob < ActiveJob::Base
@@ -178,6 +418,13 @@ See the [Logging wiki page](https://github.com/mperham/sidekiq/wiki/Logging) for
178
418
  - Integrate the StandardRB code formatter to ensure consistent code
179
419
  styling. [#4114, gearnode]
180
420
 
421
+ 5.2.10
422
+ ---------
423
+
424
+ - Backport fix for CVE-2022-23837.
425
+ - Migrate to `exists?` for redis-rb.
426
+ - Lock redis-rb to <4.6 to avoid deprecations.
427
+
181
428
  5.2.9
182
429
  ---------
183
430
 
data/LICENSE CHANGED
@@ -4,6 +4,6 @@ Sidekiq is an Open Source project licensed under the terms of
4
4
  the LGPLv3 license. Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
5
5
  for license text.
6
6
 
7
- Sidekiq Pro has a commercial-friendly license allowing private forks
8
- and modifications of Sidekiq. Please see https://sidekiq.org/products/pro.html for
9
- more detail. You can find the commercial license terms in COMM-LICENSE.
7
+ Sidekiq Pro and Sidekiq Enterprise have a commercial-friendly license.
8
+ You can find the commercial license in COMM-LICENSE.txt.
9
+ Please see https://sidekiq.org for purchasing options.
data/README.md CHANGED
@@ -2,8 +2,7 @@ Sidekiq
2
2
  ==============
3
3
 
4
4
  [![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
5
- [![Codecov](https://codecov.io/gh/mperham/sidekiq/branch/master/graph/badge.svg)](https://codecov.io/gh/mperham/sidekiq)
6
- [![Build Status](https://circleci.com/gh/mperham/sidekiq/tree/master.svg?style=svg)](https://circleci.com/gh/mperham/sidekiq/tree/master)
5
+ ![Build](https://github.com/mperham/sidekiq/workflows/CI/badge.svg)
7
6
 
8
7
  Simple, efficient background processing for Ruby.
9
8
 
@@ -37,17 +36,17 @@ Sidekiq 6.0 supports Rails 5.0+ but does not require it.
37
36
  Installation
38
37
  -----------------
39
38
 
40
- gem install sidekiq
39
+ bundle add sidekiq
41
40
 
42
41
 
43
42
  Getting Started
44
43
  -----------------
45
44
 
46
45
  See the [Getting Started wiki page](https://github.com/mperham/sidekiq/wiki/Getting-Started) and follow the simple setup process.
47
- You can watch [this Youtube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
46
+ You can watch [this YouTube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
48
47
  Sidekiq and see its features in action. Here's the Web UI:
49
48
 
50
- ![Web UI](https://github.com/mperham/sidekiq/raw/master/examples/web-ui.png)
49
+ ![Web UI](https://github.com/mperham/sidekiq/raw/main/examples/web-ui.png)
51
50
 
52
51
 
53
52
  Want to Upgrade?
@@ -81,11 +80,16 @@ Useful resources:
81
80
  Every Friday morning is Sidekiq happy hour: I video chat and answer questions.
82
81
  See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
83
82
 
83
+ Contributing
84
+ -----------------
85
+
86
+ Please see [the contributing guidelines](https://github.com/mperham/sidekiq/blob/main/.github/contributing.md).
87
+
84
88
 
85
89
  License
86
90
  -----------------
87
91
 
88
- Please see [LICENSE](https://github.com/mperham/sidekiq/blob/master/LICENSE) for licensing details.
92
+ Please see [LICENSE](https://github.com/mperham/sidekiq/blob/main/LICENSE) for licensing details.
89
93
 
90
94
 
91
95
  Author
data/bin/sidekiq CHANGED
@@ -4,7 +4,7 @@
4
4
  # RUBYOPT=-w bundle exec sidekiq
5
5
  $TESTING = false
6
6
 
7
- require_relative '../lib/sidekiq/cli'
7
+ require_relative "../lib/sidekiq/cli"
8
8
 
9
9
  def integrate_with_systemd
10
10
  return unless ENV["NOTIFY_SOCKET"]
@@ -32,8 +32,8 @@ begin
32
32
  rescue => e
33
33
  raise e if $DEBUG
34
34
  if Sidekiq.error_handlers.length == 0
35
- STDERR.puts e.message
36
- STDERR.puts e.backtrace.join("\n")
35
+ warn e.message
36
+ warn e.backtrace.join("\n")
37
37
  else
38
38
  cli.handle_exception e
39
39
  end
data/bin/sidekiqload CHANGED
@@ -4,19 +4,22 @@
4
4
  # RUBYOPT=-w bundle exec sidekiq
5
5
  $TESTING = false
6
6
 
7
- #require 'ruby-prof'
8
- require 'bundler/setup'
7
+ # require "ruby-prof"
8
+ require "bundler/setup"
9
9
  Bundler.require(:default, :load_test)
10
10
 
11
- require_relative '../lib/sidekiq/cli'
12
- require_relative '../lib/sidekiq/launcher'
11
+ require_relative "../lib/sidekiq/cli"
12
+ require_relative "../lib/sidekiq/launcher"
13
13
 
14
- include Sidekiq::Util
14
+ if ENV["SIDEKIQ_REDIS_CLIENT"]
15
+ Sidekiq::RedisConnection.adapter = :redis_client
16
+ end
15
17
 
16
18
  Sidekiq.configure_server do |config|
17
19
  config.options[:concurrency] = 10
18
- config.redis = { db: 13, port: 6380, driver: :hiredis }
19
- config.options[:queues] << 'default'
20
+ config.redis = {db: 13, port: 6380}
21
+ # config.redis = { db: 13, port: 6380, driver: :hiredis}
22
+ config.options[:queues] << "default"
20
23
  config.logger.level = Logger::ERROR
21
24
  config.average_scheduled_poll_interval = 2
22
25
  config.reliable! if defined?(Sidekiq::Pro)
@@ -29,53 +32,50 @@ class LoadWorker
29
32
  1
30
33
  end
31
34
 
32
- def perform(idx, ts=nil)
33
- puts(Time.now.to_f - ts) if ts != nil
34
- #raise idx.to_s if idx % 100 == 1
35
+ def perform(idx, ts = nil)
36
+ puts(Time.now.to_f - ts) if !ts.nil?
37
+ # raise idx.to_s if idx % 100 == 1
35
38
  end
36
39
  end
37
40
 
38
41
  # brew tap shopify/shopify
39
42
  # brew install toxiproxy
40
- # gem install toxiproxy
41
43
  # run `toxiproxy-server` in a separate terminal window.
42
- require 'toxiproxy'
44
+ require "toxiproxy"
43
45
  # simulate a non-localhost network for realer-world conditions.
44
46
  # adding 1ms of network latency has an ENORMOUS impact on benchmarks
45
47
  Toxiproxy.populate([{
46
- "name": "redis",
47
- "listen": "127.0.0.1:6380",
48
- "upstream": "127.0.0.1:6379"
48
+ name: "redis",
49
+ listen: "127.0.0.1:6380",
50
+ upstream: "127.0.0.1:6379"
49
51
  }])
50
52
 
51
53
  self_read, self_write = IO.pipe
52
- %w(INT TERM TSTP TTIN).each do |sig|
53
- begin
54
- trap sig do
55
- self_write.puts(sig)
56
- end
57
- rescue ArgumentError
58
- puts "Signal #{sig} not supported"
54
+ %w[INT TERM TSTP TTIN].each do |sig|
55
+ trap sig do
56
+ self_write.puts(sig)
59
57
  end
58
+ rescue ArgumentError
59
+ puts "Signal #{sig} not supported"
60
60
  end
61
61
 
62
- Sidekiq.redis {|c| c.flushdb}
62
+ Sidekiq.redis { |c| c.flushdb }
63
63
  def handle_signal(launcher, sig)
64
64
  Sidekiq.logger.debug "Got #{sig} signal"
65
65
  case sig
66
- when 'INT'
66
+ when "INT"
67
67
  # Handle Ctrl-C in JRuby like MRI
68
68
  # http://jira.codehaus.org/browse/JRUBY-4637
69
69
  raise Interrupt
70
- when 'TERM'
70
+ when "TERM"
71
71
  # Heroku sends TERM and then waits 30 seconds for process to exit.
72
72
  raise Interrupt
73
- when 'TSTP'
73
+ when "TSTP"
74
74
  Sidekiq.logger.info "Received TSTP, no longer accepting new work"
75
75
  launcher.quiet
76
- when 'TTIN'
76
+ when "TTIN"
77
77
  Thread.list.each do |thread|
78
- Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['label']}"
78
+ Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["label"]}"
79
79
  if thread.backtrace
80
80
  Sidekiq.logger.warn thread.backtrace.join("\n")
81
81
  else
@@ -93,49 +93,53 @@ iter = 10
93
93
  count = 10_000
94
94
 
95
95
  iter.times do
96
- arr = Array.new(count) do
97
- []
98
- end
99
- count.times do |idx|
100
- arr[idx][0] = idx
101
- end
102
- Sidekiq::Client.push_bulk('class' => LoadWorker, 'args' => arr)
96
+ arr = Array.new(count) { |idx| [idx] }
97
+ Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
103
98
  end
104
- Sidekiq.logger.error "Created #{count*iter} jobs"
99
+ Sidekiq.logger.error "Created #{count * iter} jobs"
105
100
 
106
101
  start = Time.now
107
102
 
108
103
  Monitoring = Thread.new do
109
- watchdog("monitor thread") do
110
- while true
104
+ while true
105
+ sleep 0.2
106
+ qsize = Sidekiq.redis do |conn|
107
+ conn.llen "queue:default"
108
+ end
109
+ total = qsize
110
+ # Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
111
+ if total == 0
112
+ Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
113
+ Sidekiq.logger.error("Now here's the latency for three jobs")
114
+
115
+ LoadWorker.perform_async(1, Time.now.to_f)
116
+ LoadWorker.perform_async(2, Time.now.to_f)
117
+ LoadWorker.perform_async(3, Time.now.to_f)
118
+
111
119
  sleep 0.2
112
- qsize = Sidekiq.redis do |conn|
113
- conn.llen "queue:default"
114
- end
115
- total = qsize
116
- #Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
117
- if total == 0
118
- Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
119
- Sidekiq.logger.error("Now here's the latency for three jobs")
120
-
121
- LoadWorker.perform_async(1, Time.now.to_f)
122
- LoadWorker.perform_async(2, Time.now.to_f)
123
- LoadWorker.perform_async(3, Time.now.to_f)
124
-
125
- sleep 0.2
126
- exit(0)
127
- end
120
+ exit(0)
128
121
  end
129
122
  end
130
123
  end
131
124
 
125
+ def with_latency(latency, &block)
126
+ Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
127
+ if latency > 0
128
+ Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
129
+ else
130
+ yield
131
+ end
132
+ end
133
+
132
134
  begin
133
- #RubyProf::exclude_threads = [ Monitoring ]
134
- #RubyProf.start
135
- fire_event(:startup)
136
- Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
137
- Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
138
- launcher = Sidekiq::Launcher.new(Sidekiq.options)
135
+ # RubyProf::exclude_threads = [ Monitoring ]
136
+ # RubyProf.start
137
+ events = Sidekiq.options[:lifecycle_events][:startup]
138
+ events.each(&:call)
139
+ events.clear
140
+
141
+ with_latency(Integer(ENV.fetch("LATENCY", "1"))) do
142
+ launcher = Sidekiq::Launcher.new(Sidekiq)
139
143
  launcher.run
140
144
 
141
145
  while readable_io = IO.select([self_read])
@@ -144,14 +148,14 @@ begin
144
148
  end
145
149
  end
146
150
  rescue SystemExit => e
147
- #Sidekiq.logger.error("Profiling...")
148
- #result = RubyProf.stop
149
- #printer = RubyProf::GraphHtmlPrinter.new(result)
150
- #printer.print(File.new("output.html", "w"), :min_percent => 1)
151
+ # Sidekiq.logger.error("Profiling...")
152
+ # result = RubyProf.stop
153
+ # printer = RubyProf::GraphHtmlPrinter.new(result)
154
+ # printer.print(File.new("output.html", "w"), :min_percent => 1)
151
155
  # normal
152
156
  rescue => e
153
157
  raise e if $DEBUG
154
- STDERR.puts e.message
155
- STDERR.puts e.backtrace.join("\n")
158
+ warn e.message
159
+ warn e.backtrace.join("\n")
156
160
  exit 1
157
161
  end
data/bin/sidekiqmon CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
- require 'sidekiq/monitor'
3
+ require "sidekiq/monitor"
4
4
 
5
5
  section = "all"
6
6
  section = ARGV[0] if ARGV.size == 1
@@ -0,0 +1,57 @@
1
+ require "rails/generators/named_base"
2
+
3
+ module Sidekiq
4
+ module Generators # :nodoc:
5
+ class JobGenerator < ::Rails::Generators::NamedBase # :nodoc:
6
+ desc "This generator creates a Sidekiq Job in app/sidekiq and a corresponding test"
7
+
8
+ check_class_collision suffix: "Job"
9
+
10
+ def self.default_generator_root
11
+ File.dirname(__FILE__)
12
+ end
13
+
14
+ def create_job_file
15
+ template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
16
+ end
17
+
18
+ def create_test_file
19
+ return unless test_framework
20
+
21
+ if test_framework == :rspec
22
+ create_job_spec
23
+ else
24
+ create_job_test
25
+ end
26
+ end
27
+
28
+ private
29
+
30
+ def create_job_spec
31
+ template_file = File.join(
32
+ "spec/sidekiq",
33
+ class_path,
34
+ "#{file_name}_job_spec.rb"
35
+ )
36
+ template "job_spec.rb.erb", template_file
37
+ end
38
+
39
+ def create_job_test
40
+ template_file = File.join(
41
+ "test/sidekiq",
42
+ class_path,
43
+ "#{file_name}_job_test.rb"
44
+ )
45
+ template "job_test.rb.erb", template_file
46
+ end
47
+
48
+ def file_name
49
+ @_file_name ||= super.sub(/_?job\z/i, "")
50
+ end
51
+
52
+ def test_framework
53
+ ::Rails.application.config.generators.options[:rails][:test_framework]
54
+ end
55
+ end
56
+ end
57
+ end
@@ -1,6 +1,6 @@
1
1
  <% module_namespacing do -%>
2
- class <%= class_name %>Worker
3
- include Sidekiq::Worker
2
+ class <%= class_name %>Job
3
+ include Sidekiq::Job
4
4
 
5
5
  def perform(*args)
6
6
  # Do something
@@ -1,6 +1,6 @@
1
1
  require 'rails_helper'
2
2
  <% module_namespacing do -%>
3
- RSpec.describe <%= class_name %>Worker, type: :worker do
3
+ RSpec.describe <%= class_name %>Job, type: :job do
4
4
  pending "add some examples to (or delete) #{__FILE__}"
5
5
  end
6
6
  <% end -%>
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
  <% module_namespacing do -%>
3
- class <%= class_name %>WorkerTest < Minitest::Test
3
+ class <%= class_name %>JobTest < Minitest::Test
4
4
  def test_example
5
5
  skip "add some examples to (or delete) #{__FILE__}"
6
6
  end