sidekiq 6.1.1 → 6.5.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Changes.md +250 -3
- data/LICENSE +3 -3
- data/README.md +10 -6
- data/bin/sidekiq +3 -3
- data/bin/sidekiqload +70 -66
- data/bin/sidekiqmon +1 -1
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +352 -156
- data/lib/sidekiq/cli.rb +86 -41
- data/lib/sidekiq/client.rb +49 -73
- data/lib/sidekiq/{util.rb → component.rb} +12 -14
- data/lib/sidekiq/delay.rb +3 -1
- data/lib/sidekiq/extensions/action_mailer.rb +3 -2
- data/lib/sidekiq/extensions/active_record.rb +1 -1
- data/lib/sidekiq/extensions/generic_proxy.rb +4 -2
- data/lib/sidekiq/fetch.rb +31 -20
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +16 -28
- data/lib/sidekiq/job_retry.rb +79 -59
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +126 -65
- data/lib/sidekiq/logger.rb +11 -20
- data/lib/sidekiq/manager.rb +35 -34
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +88 -42
- data/lib/sidekiq/middleware/current_attributes.rb +63 -0
- data/lib/sidekiq/middleware/i18n.rb +6 -4
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +2 -2
- data/lib/sidekiq/paginator.rb +17 -9
- data/lib/sidekiq/processor.rb +47 -41
- data/lib/sidekiq/rails.rb +32 -4
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +84 -55
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +96 -32
- data/lib/sidekiq/testing/inline.rb +4 -4
- data/lib/sidekiq/testing.rb +38 -39
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +3 -3
- data/lib/sidekiq/web/application.rb +41 -16
- data/lib/sidekiq/web/csrf_protection.rb +32 -5
- data/lib/sidekiq/web/helpers.rb +52 -30
- data/lib/sidekiq/web/router.rb +4 -1
- data/lib/sidekiq/web.rb +38 -78
- data/lib/sidekiq/worker.rb +142 -16
- data/lib/sidekiq.rb +114 -31
- data/sidekiq.gemspec +12 -4
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +114 -60
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +50 -67
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application-dark.css +61 -51
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +84 -243
- data/web/locales/ar.yml +8 -2
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +11 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +8 -1
- data/web/locales/ja.yml +10 -0
- data/web/locales/lt.yml +1 -1
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/locales/zh-cn.yml +36 -11
- data/web/locales/zh-tw.yml +32 -7
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +1 -1
- data/web/views/_nav.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +57 -21
- data/web/views/dashboard.erb +23 -14
- data/web/views/dead.erb +1 -1
- data/web/views/layout.erb +2 -1
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/morgue.erb +6 -6
- data/web/views/queue.erb +15 -11
- data/web/views/queues.erb +4 -4
- data/web/views/retries.erb +7 -7
- data/web/views/retry.erb +1 -1
- data/web/views/scheduled.erb +1 -1
- metadata +52 -39
- data/.circleci/config.yml +0 -71
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.standard.yml +0 -20
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/6.0-Upgrade.md +0 -72
- data/COMM-LICENSE +0 -97
- data/Ent-2.0-Upgrade.md +0 -37
- data/Ent-Changes.md +0 -275
- data/Gemfile +0 -24
- data/Gemfile.lock +0 -208
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-5.0-Upgrade.md +0 -25
- data/Pro-Changes.md +0 -795
- data/Rakefile +0 -10
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
- data/lib/sidekiq/exception_handler.rb +0 -27
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b12bc33857058ca07b776ff3f9e39eb98a84533c8f1c78367626a6a514d53607
|
4
|
+
data.tar.gz: d783f3060fc348786c7128ffc3b14703d9ee676a1a4972d88bc2e995fbabfd16
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0471f10240cbf2a1e2337e50111690a32153da3fa23f3e8462ff545dcc9cde06d67574cc26ed1c2be3fed88d31a34002ec30641a6c40a4f668a44005e9d5d297
|
7
|
+
data.tar.gz: 88f36a02144dbdf6fa58eb5d58f9615d99321049a76717dd2a4bd03a0bc7ebfe69e4709b860afd22782c80e5a868d8c85026d15ce42824675e3f6f09534c6335
|
data/Changes.md
CHANGED
@@ -1,8 +1,248 @@
|
|
1
1
|
# Sidekiq Changes
|
2
2
|
|
3
|
-
[Sidekiq Changes](https://github.com/mperham/sidekiq/blob/
|
3
|
+
[Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
|
4
4
|
|
5
|
-
|
5
|
+
6.5.10
|
6
|
+
----------
|
7
|
+
|
8
|
+
- Web UI DoS vector [#6045] CVE-2023-26141
|
9
|
+
- Fix broadcast logger with Rails 7.1 [#6054]
|
10
|
+
|
11
|
+
6.5.9
|
12
|
+
----------
|
13
|
+
|
14
|
+
- Ensure Sidekiq.options[:environment] == RAILS_ENV [#5932]
|
15
|
+
|
16
|
+
6.5.8
|
17
|
+
----------
|
18
|
+
|
19
|
+
- Fail if using a bad version of scout_apm [#5616]
|
20
|
+
- Add pagination to Busy page [#5556]
|
21
|
+
- Speed up WorkSet#each [#5559]
|
22
|
+
- Adjust CurrentAttributes to work with the String class name so we aren't referencing
|
23
|
+
the Class within a Rails initializer [#5536]
|
24
|
+
|
25
|
+
6.5.7
|
26
|
+
----------
|
27
|
+
|
28
|
+
- Updates for JA and ZH locales
|
29
|
+
- Further optimizations for scheduled polling [#5513]
|
30
|
+
|
31
|
+
6.5.6
|
32
|
+
----------
|
33
|
+
|
34
|
+
- Fix deprecation warnings with redis-rb 4.8.0 [#5484]
|
35
|
+
- Lock redis-rb to < 5.0 as we are moving to redis-client in Sidekiq 7.0
|
36
|
+
|
37
|
+
6.5.5
|
38
|
+
----------
|
39
|
+
|
40
|
+
- Fix require issue with job_retry.rb [#5462]
|
41
|
+
- Improve Sidekiq::Web compatibility with Rack 3.x
|
42
|
+
|
43
|
+
6.5.4
|
44
|
+
----------
|
45
|
+
|
46
|
+
- Fix invalid code on Ruby 2.5 [#5460]
|
47
|
+
- Fix further metrics dependency issues [#5457]
|
48
|
+
|
49
|
+
6.5.3
|
50
|
+
----------
|
51
|
+
|
52
|
+
- Don't require metrics code without explicit opt-in [#5456]
|
53
|
+
|
54
|
+
6.5.2
|
55
|
+
----------
|
56
|
+
|
57
|
+
- [Job Metrics are under active development, help wanted!](https://github.com/mperham/sidekiq/wiki/Metrics#contributing) **BETA**
|
58
|
+
- Add `Context` column on queue page which shows any CurrentAttributes [#5450]
|
59
|
+
- `sidekiq_retry_in` may now return `:discard` or `:kill` to dynamically stop job retries [#5406]
|
60
|
+
- Smarter sorting of processes in /busy Web UI [#5398]
|
61
|
+
- Fix broken hamburger menu in mobile UI [#5428]
|
62
|
+
- Require redis-rb 4.5.0. Note that Sidekiq will break if you use the
|
63
|
+
[`Redis.exists_returns_integer = false`](https://github.com/redis/redis-rb/blob/master/CHANGELOG.md#450) flag. [#5394]
|
64
|
+
|
65
|
+
6.5.1
|
66
|
+
----------
|
67
|
+
|
68
|
+
- Fix `push_bulk` breakage [#5387]
|
69
|
+
|
70
|
+
6.5.0
|
71
|
+
---------
|
72
|
+
|
73
|
+
- Substantial refactoring of Sidekiq server internals, part of a larger effort
|
74
|
+
to reduce Sidekiq's internal usage of global methods and data, see [docs/global_to_local.md](docs/global_to_local.md) and [docs/middleware.md](docs/middleware.md).
|
75
|
+
- **Add beta support for the `redis-client` gem**. This will become the default Redis driver in Sidekiq 7.0. [#5298]
|
76
|
+
Read more: https://github.com/mperham/sidekiq/wiki/Using-redis-client
|
77
|
+
- **Add beta support for DB transaction-aware client** [#5291]
|
78
|
+
Add this line to your initializer and any jobs created during a transaction
|
79
|
+
will only be pushed to Redis **after the transaction commits**. You will need to add the
|
80
|
+
`after_commit_everywhere` gem to your Gemfile.
|
81
|
+
```ruby
|
82
|
+
Sidekiq.transactional_push!
|
83
|
+
```
|
84
|
+
This feature does not have a lot of production usage yet; please try it out and let us
|
85
|
+
know if you have any issues. It will be fully supported in Sidekiq 7.0 or removed if it
|
86
|
+
proves problematic.
|
87
|
+
- Fix regression with middleware arguments [#5312]
|
88
|
+
|
89
|
+
6.4.2
|
90
|
+
---------
|
91
|
+
|
92
|
+
- Strict argument checking now runs after client-side middleware [#5246]
|
93
|
+
- Fix page events with live polling [#5184]
|
94
|
+
- Many under-the-hood changes to remove all usage of the term "worker"
|
95
|
+
from the Sidekiq codebase and APIs. This mostly involved RDoc and local
|
96
|
+
variable names but a few constants and public APIs were changed. The old
|
97
|
+
APIs will be removed in Sidekiq 7.0.
|
98
|
+
```
|
99
|
+
Sidekiq::DEFAULT_WORKER_OPTIONS -> Sidekiq.default_job_options
|
100
|
+
Sidekiq.default_worker_options -> Sidekiq.default_job_options
|
101
|
+
Sidekiq::Queues["default"].jobs_by_worker(HardJob) -> Sidekiq::Queues["default"].jobs_by_class(HardJob)
|
102
|
+
```
|
103
|
+
|
104
|
+
6.4.1
|
105
|
+
---------
|
106
|
+
|
107
|
+
- Fix pipeline/multi deprecations in redis-rb 4.6
|
108
|
+
- Fix sidekiq.yml YAML load errors on Ruby 3.1 [#5141]
|
109
|
+
- Sharding support for `perform_bulk` [#5129]
|
110
|
+
- Refactor job logger for SPEEEEEEED
|
111
|
+
|
112
|
+
6.4.0
|
113
|
+
---------
|
114
|
+
|
115
|
+
- **SECURITY**: Validate input to avoid possible DoS in Web UI.
|
116
|
+
- Add **strict argument checking** [#5071]
|
117
|
+
Sidekiq will now log a warning if JSON-unsafe arguments are passed to `perform_async`.
|
118
|
+
Add `Sidekiq.strict_args!(false)` to your initializer to disable this warning.
|
119
|
+
This warning will switch to an exception in Sidekiq 7.0.
|
120
|
+
- Note that Delayed Extensions will be removed in Sidekiq 7.0 [#5076]
|
121
|
+
- Add `perform_{inline,sync}` in Sidekiq::Job to run a job synchronously [#5061, hasan-ally]
|
122
|
+
```ruby
|
123
|
+
SomeJob.perform_async(args...)
|
124
|
+
SomeJob.perform_sync(args...)
|
125
|
+
SomeJob.perform_inline(args...)
|
126
|
+
```
|
127
|
+
You can also dynamically redirect a job to run synchronously:
|
128
|
+
```ruby
|
129
|
+
SomeJob.set("sync": true).perform_async(args...) # will run via perform_inline
|
130
|
+
```
|
131
|
+
- Replace Sidekiq::Worker `app/workers` generator with Sidekiq::Job `app/sidekiq` generator [#5055]
|
132
|
+
```
|
133
|
+
bin/rails generate sidekiq:job ProcessOrderJob
|
134
|
+
```
|
135
|
+
- Fix job retries losing CurrentAttributes [#5090]
|
136
|
+
- Tweak shutdown to give long-running threads time to cleanup [#5095]
|
137
|
+
|
138
|
+
6.3.1
|
139
|
+
---------
|
140
|
+
|
141
|
+
- Fix keyword arguments error with CurrentAttributes on Ruby 3.0 [#5048]
|
142
|
+
|
143
|
+
6.3.0
|
144
|
+
---------
|
145
|
+
|
146
|
+
- **BREAK**: The Web UI has been refactored to remove jQuery. Any UI extensions
|
147
|
+
which use jQuery will break.
|
148
|
+
- **FEATURE**: Sidekiq.logger has been enhanced so any `Rails.logger`
|
149
|
+
output in jobs now shows up in the Sidekiq console. Remove any logger
|
150
|
+
hacks in your initializer and see if it Just Works™ now. [#5021]
|
151
|
+
- **FEATURE**: Add `Sidekiq::Job` alias for `Sidekiq::Worker`, to better
|
152
|
+
reflect industry standard terminology. You can now do this:
|
153
|
+
```ruby
|
154
|
+
class MyJob
|
155
|
+
include Sidekiq::Job
|
156
|
+
sidekiq_options ...
|
157
|
+
def perform(args)
|
158
|
+
end
|
159
|
+
end
|
160
|
+
```
|
161
|
+
- **FEATURE**: Support for serializing ActiveSupport::CurrentAttributes into each job. [#4982]
|
162
|
+
```ruby
|
163
|
+
# config/initializers/sidekiq.rb
|
164
|
+
require "sidekiq/middleware/current_attributes"
|
165
|
+
Sidekiq::CurrentAttributes.persist(Myapp::Current) # Your AS::CurrentAttributes singleton
|
166
|
+
```
|
167
|
+
- **FEATURE**: Add `Sidekiq::Worker.perform_bulk` for enqueuing jobs in bulk,
|
168
|
+
similar to `Sidekiq::Client.push_bulk` [#5042]
|
169
|
+
```ruby
|
170
|
+
MyJob.perform_bulk([[1], [2], [3]])
|
171
|
+
```
|
172
|
+
- Implement `queue_as`, `wait` and `wait_until` for ActiveJob compatibility [#5003]
|
173
|
+
- Scheduler now uses Lua to reduce Redis load and network roundtrips [#5044]
|
174
|
+
- Retry Redis operation if we get an `UNBLOCKED` Redis error [#4985]
|
175
|
+
- Run existing signal traps, if any, before running Sidekiq's trap [#4991]
|
176
|
+
- Fix fetch bug when using weighted queues which caused Sidekiq to stop
|
177
|
+
processing queues randomly [#5031]
|
178
|
+
|
179
|
+
6.2.2
|
180
|
+
---------
|
181
|
+
|
182
|
+
- Reduce retry jitter, add jitter to `sidekiq_retry_in` values [#4957]
|
183
|
+
- Minimize scheduler load on Redis at scale [#4882]
|
184
|
+
- Improve logging of delay jobs [#4904, BuonOno]
|
185
|
+
- Minor CSS improvements for buttons and tables, design PRs always welcome!
|
186
|
+
- Tweak Web UI `Cache-Control` header [#4966]
|
187
|
+
- Rename internal API class `Sidekiq::Job` to `Sidekiq::JobRecord` [#4955]
|
188
|
+
|
189
|
+
6.2.1
|
190
|
+
---------
|
191
|
+
|
192
|
+
- Update RTT warning logic to handle transient RTT spikes [#4851]
|
193
|
+
- Fix very low priority CVE on unescaped queue name [#4852]
|
194
|
+
- Add note about sessions and Rails apps in API mode
|
195
|
+
|
196
|
+
6.2.0
|
197
|
+
---------
|
198
|
+
|
199
|
+
- Store Redis RTT and log if poor [#4824]
|
200
|
+
- Add process/thread stats to Busy page [#4806]
|
201
|
+
- Improve Web UI on mobile devices [#4840]
|
202
|
+
- **Refactor Web UI session usage** [#4804]
|
203
|
+
Numerous people have hit "Forbidden" errors and struggled with Sidekiq's
|
204
|
+
Web UI session requirement. If you have code in your initializer for
|
205
|
+
Web sessions, it's quite possible it will need to be removed. Here's
|
206
|
+
an overview:
|
207
|
+
```
|
208
|
+
Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app,
|
209
|
+
make sure you mount Sidekiq::Web *inside* your routes in `config/routes.rb` so
|
210
|
+
Sidekiq can reuse the Rails session:
|
211
|
+
|
212
|
+
Rails.application.routes.draw do
|
213
|
+
mount Sidekiq::Web => "/sidekiq"
|
214
|
+
....
|
215
|
+
end
|
216
|
+
|
217
|
+
If this is a bare Rack app, use a session middleware before Sidekiq::Web:
|
218
|
+
|
219
|
+
# first, use IRB to create a shared secret key for sessions and commit it
|
220
|
+
require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
|
221
|
+
|
222
|
+
# now, update your Rack app to include the secret with a session cookie middleware
|
223
|
+
use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
|
224
|
+
run Sidekiq::Web
|
225
|
+
|
226
|
+
If this is a Rails app in API mode, you need to enable sessions.
|
227
|
+
|
228
|
+
https://guides.rubyonrails.org/api_app.html#using-session-middlewares
|
229
|
+
```
|
230
|
+
|
231
|
+
6.1.3
|
232
|
+
---------
|
233
|
+
|
234
|
+
- Warn if Redis is configured to evict data under memory pressure [#4752]
|
235
|
+
- Add process RSS on the Busy page [#4717]
|
236
|
+
|
237
|
+
6.1.2
|
238
|
+
---------
|
239
|
+
|
240
|
+
- Improve readability in dark mode Web UI [#4674]
|
241
|
+
- Fix Web UI crash with corrupt session [#4672]
|
242
|
+
- Allow middleware to yield arguments [#4673, @eugeneius]
|
243
|
+
- Migrate CI from CircleCI to GitHub Actions [#4677]
|
244
|
+
|
245
|
+
6.1.1
|
6
246
|
---------
|
7
247
|
|
8
248
|
- Jobs are now sorted by age in the Busy Workers table. [#4641]
|
@@ -142,7 +382,7 @@ assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs_for(FooMailer).size
|
|
142
382
|
|
143
383
|
This release has major breaking changes. Read and test carefully in production.
|
144
384
|
|
145
|
-
- With Rails 6.0.
|
385
|
+
- With Rails 6.0.2+, ActiveJobs can now use `sidekiq_options` directly to configure Sidekiq
|
146
386
|
features/internals like the retry subsystem. [#4213, pirj]
|
147
387
|
```ruby
|
148
388
|
class MyJob < ActiveJob::Base
|
@@ -178,6 +418,13 @@ See the [Logging wiki page](https://github.com/mperham/sidekiq/wiki/Logging) for
|
|
178
418
|
- Integrate the StandardRB code formatter to ensure consistent code
|
179
419
|
styling. [#4114, gearnode]
|
180
420
|
|
421
|
+
5.2.10
|
422
|
+
---------
|
423
|
+
|
424
|
+
- Backport fix for CVE-2022-23837.
|
425
|
+
- Migrate to `exists?` for redis-rb.
|
426
|
+
- Lock redis-rb to <4.6 to avoid deprecations.
|
427
|
+
|
181
428
|
5.2.9
|
182
429
|
---------
|
183
430
|
|
data/LICENSE
CHANGED
@@ -4,6 +4,6 @@ Sidekiq is an Open Source project licensed under the terms of
|
|
4
4
|
the LGPLv3 license. Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
|
5
5
|
for license text.
|
6
6
|
|
7
|
-
Sidekiq Pro
|
8
|
-
|
9
|
-
|
7
|
+
Sidekiq Pro and Sidekiq Enterprise have a commercial-friendly license.
|
8
|
+
You can find the commercial license in COMM-LICENSE.txt.
|
9
|
+
Please see https://sidekiq.org for purchasing options.
|
data/README.md
CHANGED
@@ -2,8 +2,7 @@ Sidekiq
|
|
2
2
|
==============
|
3
3
|
|
4
4
|
[![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
|
5
|
-
|
6
|
-
[![Build Status](https://circleci.com/gh/mperham/sidekiq/tree/master.svg?style=svg)](https://circleci.com/gh/mperham/sidekiq/tree/master)
|
5
|
+
![Build](https://github.com/mperham/sidekiq/workflows/CI/badge.svg)
|
7
6
|
|
8
7
|
Simple, efficient background processing for Ruby.
|
9
8
|
|
@@ -37,17 +36,17 @@ Sidekiq 6.0 supports Rails 5.0+ but does not require it.
|
|
37
36
|
Installation
|
38
37
|
-----------------
|
39
38
|
|
40
|
-
|
39
|
+
bundle add sidekiq
|
41
40
|
|
42
41
|
|
43
42
|
Getting Started
|
44
43
|
-----------------
|
45
44
|
|
46
45
|
See the [Getting Started wiki page](https://github.com/mperham/sidekiq/wiki/Getting-Started) and follow the simple setup process.
|
47
|
-
You can watch [this
|
46
|
+
You can watch [this YouTube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
|
48
47
|
Sidekiq and see its features in action. Here's the Web UI:
|
49
48
|
|
50
|
-
![Web UI](https://github.com/mperham/sidekiq/raw/
|
49
|
+
![Web UI](https://github.com/mperham/sidekiq/raw/main/examples/web-ui.png)
|
51
50
|
|
52
51
|
|
53
52
|
Want to Upgrade?
|
@@ -81,11 +80,16 @@ Useful resources:
|
|
81
80
|
Every Friday morning is Sidekiq happy hour: I video chat and answer questions.
|
82
81
|
See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
|
83
82
|
|
83
|
+
Contributing
|
84
|
+
-----------------
|
85
|
+
|
86
|
+
Please see [the contributing guidelines](https://github.com/mperham/sidekiq/blob/main/.github/contributing.md).
|
87
|
+
|
84
88
|
|
85
89
|
License
|
86
90
|
-----------------
|
87
91
|
|
88
|
-
Please see [LICENSE](https://github.com/mperham/sidekiq/blob/
|
92
|
+
Please see [LICENSE](https://github.com/mperham/sidekiq/blob/main/LICENSE) for licensing details.
|
89
93
|
|
90
94
|
|
91
95
|
Author
|
data/bin/sidekiq
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
# RUBYOPT=-w bundle exec sidekiq
|
5
5
|
$TESTING = false
|
6
6
|
|
7
|
-
require_relative
|
7
|
+
require_relative "../lib/sidekiq/cli"
|
8
8
|
|
9
9
|
def integrate_with_systemd
|
10
10
|
return unless ENV["NOTIFY_SOCKET"]
|
@@ -32,8 +32,8 @@ begin
|
|
32
32
|
rescue => e
|
33
33
|
raise e if $DEBUG
|
34
34
|
if Sidekiq.error_handlers.length == 0
|
35
|
-
|
36
|
-
|
35
|
+
warn e.message
|
36
|
+
warn e.backtrace.join("\n")
|
37
37
|
else
|
38
38
|
cli.handle_exception e
|
39
39
|
end
|
data/bin/sidekiqload
CHANGED
@@ -4,19 +4,22 @@
|
|
4
4
|
# RUBYOPT=-w bundle exec sidekiq
|
5
5
|
$TESTING = false
|
6
6
|
|
7
|
-
#require
|
8
|
-
require
|
7
|
+
# require "ruby-prof"
|
8
|
+
require "bundler/setup"
|
9
9
|
Bundler.require(:default, :load_test)
|
10
10
|
|
11
|
-
require_relative
|
12
|
-
require_relative
|
11
|
+
require_relative "../lib/sidekiq/cli"
|
12
|
+
require_relative "../lib/sidekiq/launcher"
|
13
13
|
|
14
|
-
|
14
|
+
if ENV["SIDEKIQ_REDIS_CLIENT"]
|
15
|
+
Sidekiq::RedisConnection.adapter = :redis_client
|
16
|
+
end
|
15
17
|
|
16
18
|
Sidekiq.configure_server do |config|
|
17
19
|
config.options[:concurrency] = 10
|
18
|
-
config.redis = {
|
19
|
-
config.
|
20
|
+
config.redis = {db: 13, port: 6380}
|
21
|
+
# config.redis = { db: 13, port: 6380, driver: :hiredis}
|
22
|
+
config.options[:queues] << "default"
|
20
23
|
config.logger.level = Logger::ERROR
|
21
24
|
config.average_scheduled_poll_interval = 2
|
22
25
|
config.reliable! if defined?(Sidekiq::Pro)
|
@@ -29,53 +32,50 @@ class LoadWorker
|
|
29
32
|
1
|
30
33
|
end
|
31
34
|
|
32
|
-
def perform(idx, ts=nil)
|
33
|
-
puts(Time.now.to_f - ts) if ts
|
34
|
-
#raise idx.to_s if idx % 100 == 1
|
35
|
+
def perform(idx, ts = nil)
|
36
|
+
puts(Time.now.to_f - ts) if !ts.nil?
|
37
|
+
# raise idx.to_s if idx % 100 == 1
|
35
38
|
end
|
36
39
|
end
|
37
40
|
|
38
41
|
# brew tap shopify/shopify
|
39
42
|
# brew install toxiproxy
|
40
|
-
# gem install toxiproxy
|
41
43
|
# run `toxiproxy-server` in a separate terminal window.
|
42
|
-
require
|
44
|
+
require "toxiproxy"
|
43
45
|
# simulate a non-localhost network for realer-world conditions.
|
44
46
|
# adding 1ms of network latency has an ENORMOUS impact on benchmarks
|
45
47
|
Toxiproxy.populate([{
|
46
|
-
|
47
|
-
|
48
|
-
|
48
|
+
name: "redis",
|
49
|
+
listen: "127.0.0.1:6380",
|
50
|
+
upstream: "127.0.0.1:6379"
|
49
51
|
}])
|
50
52
|
|
51
53
|
self_read, self_write = IO.pipe
|
52
|
-
%w
|
53
|
-
|
54
|
-
|
55
|
-
self_write.puts(sig)
|
56
|
-
end
|
57
|
-
rescue ArgumentError
|
58
|
-
puts "Signal #{sig} not supported"
|
54
|
+
%w[INT TERM TSTP TTIN].each do |sig|
|
55
|
+
trap sig do
|
56
|
+
self_write.puts(sig)
|
59
57
|
end
|
58
|
+
rescue ArgumentError
|
59
|
+
puts "Signal #{sig} not supported"
|
60
60
|
end
|
61
61
|
|
62
|
-
Sidekiq.redis {|c| c.flushdb}
|
62
|
+
Sidekiq.redis { |c| c.flushdb }
|
63
63
|
def handle_signal(launcher, sig)
|
64
64
|
Sidekiq.logger.debug "Got #{sig} signal"
|
65
65
|
case sig
|
66
|
-
when
|
66
|
+
when "INT"
|
67
67
|
# Handle Ctrl-C in JRuby like MRI
|
68
68
|
# http://jira.codehaus.org/browse/JRUBY-4637
|
69
69
|
raise Interrupt
|
70
|
-
when
|
70
|
+
when "TERM"
|
71
71
|
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
72
72
|
raise Interrupt
|
73
|
-
when
|
73
|
+
when "TSTP"
|
74
74
|
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
75
75
|
launcher.quiet
|
76
|
-
when
|
76
|
+
when "TTIN"
|
77
77
|
Thread.list.each do |thread|
|
78
|
-
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread[
|
78
|
+
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["label"]}"
|
79
79
|
if thread.backtrace
|
80
80
|
Sidekiq.logger.warn thread.backtrace.join("\n")
|
81
81
|
else
|
@@ -93,49 +93,53 @@ iter = 10
|
|
93
93
|
count = 10_000
|
94
94
|
|
95
95
|
iter.times do
|
96
|
-
arr = Array.new(count)
|
97
|
-
|
98
|
-
end
|
99
|
-
count.times do |idx|
|
100
|
-
arr[idx][0] = idx
|
101
|
-
end
|
102
|
-
Sidekiq::Client.push_bulk('class' => LoadWorker, 'args' => arr)
|
96
|
+
arr = Array.new(count) { |idx| [idx] }
|
97
|
+
Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
|
103
98
|
end
|
104
|
-
Sidekiq.logger.error "Created #{count*iter} jobs"
|
99
|
+
Sidekiq.logger.error "Created #{count * iter} jobs"
|
105
100
|
|
106
101
|
start = Time.now
|
107
102
|
|
108
103
|
Monitoring = Thread.new do
|
109
|
-
|
110
|
-
|
104
|
+
while true
|
105
|
+
sleep 0.2
|
106
|
+
qsize = Sidekiq.redis do |conn|
|
107
|
+
conn.llen "queue:default"
|
108
|
+
end
|
109
|
+
total = qsize
|
110
|
+
# Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
|
111
|
+
if total == 0
|
112
|
+
Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
|
113
|
+
Sidekiq.logger.error("Now here's the latency for three jobs")
|
114
|
+
|
115
|
+
LoadWorker.perform_async(1, Time.now.to_f)
|
116
|
+
LoadWorker.perform_async(2, Time.now.to_f)
|
117
|
+
LoadWorker.perform_async(3, Time.now.to_f)
|
118
|
+
|
111
119
|
sleep 0.2
|
112
|
-
|
113
|
-
conn.llen "queue:default"
|
114
|
-
end
|
115
|
-
total = qsize
|
116
|
-
#Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
|
117
|
-
if total == 0
|
118
|
-
Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
|
119
|
-
Sidekiq.logger.error("Now here's the latency for three jobs")
|
120
|
-
|
121
|
-
LoadWorker.perform_async(1, Time.now.to_f)
|
122
|
-
LoadWorker.perform_async(2, Time.now.to_f)
|
123
|
-
LoadWorker.perform_async(3, Time.now.to_f)
|
124
|
-
|
125
|
-
sleep 0.2
|
126
|
-
exit(0)
|
127
|
-
end
|
120
|
+
exit(0)
|
128
121
|
end
|
129
122
|
end
|
130
123
|
end
|
131
124
|
|
125
|
+
def with_latency(latency, &block)
|
126
|
+
Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
|
127
|
+
if latency > 0
|
128
|
+
Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
|
129
|
+
else
|
130
|
+
yield
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
132
134
|
begin
|
133
|
-
#RubyProf::exclude_threads = [ Monitoring ]
|
134
|
-
#RubyProf.start
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
135
|
+
# RubyProf::exclude_threads = [ Monitoring ]
|
136
|
+
# RubyProf.start
|
137
|
+
events = Sidekiq.options[:lifecycle_events][:startup]
|
138
|
+
events.each(&:call)
|
139
|
+
events.clear
|
140
|
+
|
141
|
+
with_latency(Integer(ENV.fetch("LATENCY", "1"))) do
|
142
|
+
launcher = Sidekiq::Launcher.new(Sidekiq)
|
139
143
|
launcher.run
|
140
144
|
|
141
145
|
while readable_io = IO.select([self_read])
|
@@ -144,14 +148,14 @@ begin
|
|
144
148
|
end
|
145
149
|
end
|
146
150
|
rescue SystemExit => e
|
147
|
-
#Sidekiq.logger.error("Profiling...")
|
148
|
-
#result = RubyProf.stop
|
149
|
-
#printer = RubyProf::GraphHtmlPrinter.new(result)
|
150
|
-
#printer.print(File.new("output.html", "w"), :min_percent => 1)
|
151
|
+
# Sidekiq.logger.error("Profiling...")
|
152
|
+
# result = RubyProf.stop
|
153
|
+
# printer = RubyProf::GraphHtmlPrinter.new(result)
|
154
|
+
# printer.print(File.new("output.html", "w"), :min_percent => 1)
|
151
155
|
# normal
|
152
156
|
rescue => e
|
153
157
|
raise e if $DEBUG
|
154
|
-
|
155
|
-
|
158
|
+
warn e.message
|
159
|
+
warn e.backtrace.join("\n")
|
156
160
|
exit 1
|
157
161
|
end
|
data/bin/sidekiqmon
CHANGED
@@ -0,0 +1,57 @@
|
|
1
|
+
require "rails/generators/named_base"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
module Generators # :nodoc:
|
5
|
+
class JobGenerator < ::Rails::Generators::NamedBase # :nodoc:
|
6
|
+
desc "This generator creates a Sidekiq Job in app/sidekiq and a corresponding test"
|
7
|
+
|
8
|
+
check_class_collision suffix: "Job"
|
9
|
+
|
10
|
+
def self.default_generator_root
|
11
|
+
File.dirname(__FILE__)
|
12
|
+
end
|
13
|
+
|
14
|
+
def create_job_file
|
15
|
+
template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
|
16
|
+
end
|
17
|
+
|
18
|
+
def create_test_file
|
19
|
+
return unless test_framework
|
20
|
+
|
21
|
+
if test_framework == :rspec
|
22
|
+
create_job_spec
|
23
|
+
else
|
24
|
+
create_job_test
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def create_job_spec
|
31
|
+
template_file = File.join(
|
32
|
+
"spec/sidekiq",
|
33
|
+
class_path,
|
34
|
+
"#{file_name}_job_spec.rb"
|
35
|
+
)
|
36
|
+
template "job_spec.rb.erb", template_file
|
37
|
+
end
|
38
|
+
|
39
|
+
def create_job_test
|
40
|
+
template_file = File.join(
|
41
|
+
"test/sidekiq",
|
42
|
+
class_path,
|
43
|
+
"#{file_name}_job_test.rb"
|
44
|
+
)
|
45
|
+
template "job_test.rb.erb", template_file
|
46
|
+
end
|
47
|
+
|
48
|
+
def file_name
|
49
|
+
@_file_name ||= super.sub(/_?job\z/i, "")
|
50
|
+
end
|
51
|
+
|
52
|
+
def test_framework
|
53
|
+
::Rails.application.config.generators.options[:rails][:test_framework]
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|