sidekiq 6.0.7 → 6.4.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +160 -2
- data/LICENSE +3 -3
- data/README.md +5 -9
- data/bin/sidekiq +7 -2
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +104 -60
- data/lib/sidekiq/cli.rb +36 -10
- data/lib/sidekiq/client.rb +12 -45
- data/lib/sidekiq/delay.rb +2 -0
- data/lib/sidekiq/extensions/action_mailer.rb +5 -4
- data/lib/sidekiq/extensions/active_record.rb +6 -5
- data/lib/sidekiq/extensions/class_methods.rb +7 -6
- data/lib/sidekiq/extensions/generic_proxy.rb +5 -3
- data/lib/sidekiq/fetch.rb +30 -21
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +1 -1
- data/lib/sidekiq/job_retry.rb +10 -11
- data/lib/sidekiq/job_util.rb +65 -0
- data/lib/sidekiq/launcher.rb +79 -21
- data/lib/sidekiq/logger.rb +3 -2
- data/lib/sidekiq/manager.rb +10 -12
- data/lib/sidekiq/middleware/chain.rb +6 -4
- data/lib/sidekiq/middleware/current_attributes.rb +57 -0
- data/lib/sidekiq/processor.rb +4 -4
- data/lib/sidekiq/rails.rb +27 -18
- data/lib/sidekiq/redis_connection.rb +14 -13
- data/lib/sidekiq/scheduled.rb +51 -16
- data/lib/sidekiq/sd_notify.rb +1 -1
- data/lib/sidekiq/testing.rb +2 -4
- data/lib/sidekiq/util.rb +41 -0
- data/lib/sidekiq/version.rb +1 -1
- data/lib/sidekiq/web/action.rb +2 -2
- data/lib/sidekiq/web/application.rb +21 -12
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +35 -29
- data/lib/sidekiq/web/router.rb +5 -2
- data/lib/sidekiq/web.rb +36 -72
- data/lib/sidekiq/worker.rb +129 -12
- data/lib/sidekiq.rb +12 -3
- data/sidekiq.gemspec +11 -4
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +82 -66
- data/web/assets/javascripts/dashboard.js +51 -51
- data/web/assets/stylesheets/application-dark.css +64 -43
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +41 -239
- data/web/locales/ar.yml +8 -2
- data/web/locales/en.yml +4 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +8 -1
- data/web/locales/ja.yml +3 -0
- data/web/locales/lt.yml +1 -1
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +50 -19
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +1 -1
- data/web/views/layout.erb +2 -1
- data/web/views/morgue.erb +6 -6
- data/web/views/queue.erb +11 -11
- data/web/views/queues.erb +4 -4
- data/web/views/retries.erb +7 -7
- data/web/views/retry.erb +1 -1
- data/web/views/scheduled.erb +1 -1
- metadata +24 -49
- data/.circleci/config.yml +0 -60
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -13
- data/.standard.yml +0 -20
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/6.0-Upgrade.md +0 -72
- data/COMM-LICENSE +0 -97
- data/Ent-2.0-Upgrade.md +0 -37
- data/Ent-Changes.md +0 -256
- data/Gemfile +0 -24
- data/Gemfile.lock +0 -208
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-5.0-Upgrade.md +0 -25
- data/Pro-Changes.md +0 -782
- data/Rakefile +0 -10
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -57
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9622b2851203b0c5a80695ab7801ca77e15dc63a641ae79132cda9a2fcbe0cc6
|
4
|
+
data.tar.gz: dd943a02d2cf910f51866d02254f3a7845cb159735bd54d223dd7127a1fbd2fa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 65bcd542866d8699ecf5958d81a15fd322cd1c51dfc1dbc6a8b15402b70862510c134e2b0ed9f9dbcdbb4dea3a59c1d12878ff926145e503079a59896f279ff3
|
7
|
+
data.tar.gz: 36143e85dc7fd4611f8a43fe39788dc590725ac63a827fbc174916da2d59fdadb3fb64fa0d819c4ef90694f6f292ea15f7fb58fb29f368c0a06c5b90f1b4bca7
|
data/Changes.md
CHANGED
@@ -1,6 +1,158 @@
|
|
1
1
|
# Sidekiq Changes
|
2
2
|
|
3
|
-
[Sidekiq Changes](https://github.com/mperham/sidekiq/blob/
|
3
|
+
[Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
|
4
|
+
|
5
|
+
6.4.0
|
6
|
+
---------
|
7
|
+
|
8
|
+
- **SECURITY**: Validate input to avoid possible DoS in Web UI.
|
9
|
+
- Add **strict argument checking** [#5071]
|
10
|
+
Sidekiq will now log a warning if JSON-unsafe arguments are passed to `perform_async`.
|
11
|
+
Add `Sidekiq.strict_args!(false)` to your initializer to disable this warning.
|
12
|
+
This warning will switch to an exception in Sidekiq 7.0.
|
13
|
+
- Note that Delayed Extensions will be removed in Sidekiq 7.0 [#5076]
|
14
|
+
- Add `perform_{inline,sync}` in Sidekiq::Job to run a job synchronously [#5061, hasan-ally]
|
15
|
+
```ruby
|
16
|
+
SomeJob.perform_async(args...)
|
17
|
+
SomeJob.perform_sync(args...)
|
18
|
+
SomeJob.perform_inline(args...)
|
19
|
+
```
|
20
|
+
You can also dynamically redirect a job to run synchronously:
|
21
|
+
```ruby
|
22
|
+
SomeJob.set("sync": true).perform_async(args...) # will run via perform_inline
|
23
|
+
```
|
24
|
+
- Replace Sidekiq::Worker `app/workers` generator with Sidekiq::Job `app/sidekiq` generator [#5055]
|
25
|
+
```
|
26
|
+
bin/rails generate sidekiq:job ProcessOrderJob
|
27
|
+
```
|
28
|
+
- Fix job retries losing CurrentAttributes [#5090]
|
29
|
+
- Tweak shutdown to give long-running threads time to cleanup [#5095]
|
30
|
+
- Add keyword arguments support in extensions
|
31
|
+
|
32
|
+
6.3.1
|
33
|
+
---------
|
34
|
+
|
35
|
+
- Fix keyword arguments error with CurrentAttributes on Ruby 3.0 [#5048]
|
36
|
+
|
37
|
+
6.3.0
|
38
|
+
---------
|
39
|
+
|
40
|
+
- **BREAK**: The Web UI has been refactored to remove jQuery. Any UI extensions
|
41
|
+
which use jQuery will break.
|
42
|
+
- **FEATURE**: Sidekiq.logger has been enhanced so any `Rails.logger`
|
43
|
+
output in jobs now shows up in the Sidekiq console. Remove any logger
|
44
|
+
hacks in your initializer and see if it Just Works™ now. [#5021]
|
45
|
+
- **FEATURE**: Add `Sidekiq::Job` alias for `Sidekiq::Worker`, to better
|
46
|
+
reflect industry standard terminology. You can now do this:
|
47
|
+
```ruby
|
48
|
+
class MyJob
|
49
|
+
include Sidekiq::Job
|
50
|
+
sidekiq_options ...
|
51
|
+
def perform(args)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
```
|
55
|
+
- **FEATURE**: Support for serializing ActiveSupport::CurrentAttributes into each job. [#4982]
|
56
|
+
```ruby
|
57
|
+
# config/initializers/sidekiq.rb
|
58
|
+
require "sidekiq/middleware/current_attributes"
|
59
|
+
Sidekiq::CurrentAttributes.persist(Myapp::Current) # Your AS::CurrentAttributes singleton
|
60
|
+
```
|
61
|
+
- **FEATURE**: Add `Sidekiq::Worker.perform_bulk` for enqueuing jobs in bulk,
|
62
|
+
similar to `Sidekiq::Client.push_bulk` [#5042]
|
63
|
+
```ruby
|
64
|
+
MyJob.perform_bulk([[1], [2], [3]])
|
65
|
+
```
|
66
|
+
- Implement `queue_as`, `wait` and `wait_until` for ActiveJob compatibility [#5003]
|
67
|
+
- Scheduler now uses Lua to reduce Redis load and network roundtrips [#5044]
|
68
|
+
- Retry Redis operation if we get an `UNBLOCKED` Redis error [#4985]
|
69
|
+
- Run existing signal traps, if any, before running Sidekiq's trap [#4991]
|
70
|
+
- Fix fetch bug when using weighted queues which caused Sidekiq to stop
|
71
|
+
processing queues randomly [#5031]
|
72
|
+
|
73
|
+
6.2.2
|
74
|
+
---------
|
75
|
+
|
76
|
+
- Reduce retry jitter, add jitter to `sidekiq_retry_in` values [#4957]
|
77
|
+
- Minimize scheduler load on Redis at scale [#4882]
|
78
|
+
- Improve logging of delay jobs [#4904, BuonOno]
|
79
|
+
- Minor CSS improvements for buttons and tables, design PRs always welcome!
|
80
|
+
- Tweak Web UI `Cache-Control` header [#4966]
|
81
|
+
- Rename internal API class `Sidekiq::Job` to `Sidekiq::JobRecord` [#4955]
|
82
|
+
|
83
|
+
6.2.1
|
84
|
+
---------
|
85
|
+
|
86
|
+
- Update RTT warning logic to handle transient RTT spikes [#4851]
|
87
|
+
- Fix very low priority CVE on unescaped queue name [#4852]
|
88
|
+
- Add note about sessions and Rails apps in API mode
|
89
|
+
|
90
|
+
6.2.0
|
91
|
+
---------
|
92
|
+
|
93
|
+
- Store Redis RTT and log if poor [#4824]
|
94
|
+
- Add process/thread stats to Busy page [#4806]
|
95
|
+
- Improve Web UI on mobile devices [#4840]
|
96
|
+
- **Refactor Web UI session usage** [#4804]
|
97
|
+
Numerous people have hit "Forbidden" errors and struggled with Sidekiq's
|
98
|
+
Web UI session requirement. If you have code in your initializer for
|
99
|
+
Web sessions, it's quite possible it will need to be removed. Here's
|
100
|
+
an overview:
|
101
|
+
```
|
102
|
+
Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app,
|
103
|
+
make sure you mount Sidekiq::Web *inside* your routes in `config/routes.rb` so
|
104
|
+
Sidekiq can reuse the Rails session:
|
105
|
+
|
106
|
+
Rails.application.routes.draw do
|
107
|
+
mount Sidekiq::Web => "/sidekiq"
|
108
|
+
....
|
109
|
+
end
|
110
|
+
|
111
|
+
If this is a bare Rack app, use a session middleware before Sidekiq::Web:
|
112
|
+
|
113
|
+
# first, use IRB to create a shared secret key for sessions and commit it
|
114
|
+
require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
|
115
|
+
|
116
|
+
# now, update your Rack app to include the secret with a session cookie middleware
|
117
|
+
use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
|
118
|
+
run Sidekiq::Web
|
119
|
+
|
120
|
+
If this is a Rails app in API mode, you need to enable sessions.
|
121
|
+
|
122
|
+
https://guides.rubyonrails.org/api_app.html#using-session-middlewares
|
123
|
+
```
|
124
|
+
|
125
|
+
6.1.3
|
126
|
+
---------
|
127
|
+
|
128
|
+
- Warn if Redis is configured to evict data under memory pressure [#4752]
|
129
|
+
- Add process RSS on the Busy page [#4717]
|
130
|
+
|
131
|
+
6.1.2
|
132
|
+
---------
|
133
|
+
|
134
|
+
- Improve readability in dark mode Web UI [#4674]
|
135
|
+
- Fix Web UI crash with corrupt session [#4672]
|
136
|
+
- Allow middleware to yield arguments [#4673, @eugeneius]
|
137
|
+
- Migrate CI from CircleCI to GitHub Actions [#4677]
|
138
|
+
|
139
|
+
6.1.1
|
140
|
+
---------
|
141
|
+
|
142
|
+
- Jobs are now sorted by age in the Busy Workers table. [#4641]
|
143
|
+
- Fix "check all" JS logic in Web UI [#4619]
|
144
|
+
|
145
|
+
6.1.0
|
146
|
+
---------
|
147
|
+
|
148
|
+
- Web UI - Dark Mode fixes [#4543, natematykiewicz]
|
149
|
+
- Ensure `Rack::ContentLength` is loaded as middleware for correct Web UI responses [#4541]
|
150
|
+
- Avoid exception dumping SSL store in Redis connection logging [#4532]
|
151
|
+
- Better error messages in Sidekiq::Client [#4549]
|
152
|
+
- Remove rack-protection, reimplement CSRF protection [#4588]
|
153
|
+
- Require redis-rb 4.2 [#4591]
|
154
|
+
- Update to jquery 1.12.4 [#4593]
|
155
|
+
- Refactor internal fetch logic and API [#4602]
|
4
156
|
|
5
157
|
6.0.7
|
6
158
|
---------
|
@@ -124,7 +276,7 @@ assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs_for(FooMailer).size
|
|
124
276
|
|
125
277
|
This release has major breaking changes. Read and test carefully in production.
|
126
278
|
|
127
|
-
- With Rails 6.0.
|
279
|
+
- With Rails 6.0.2+, ActiveJobs can now use `sidekiq_options` directly to configure Sidekiq
|
128
280
|
features/internals like the retry subsystem. [#4213, pirj]
|
129
281
|
```ruby
|
130
282
|
class MyJob < ActiveJob::Base
|
@@ -160,6 +312,12 @@ See the [Logging wiki page](https://github.com/mperham/sidekiq/wiki/Logging) for
|
|
160
312
|
- Integrate the StandardRB code formatter to ensure consistent code
|
161
313
|
styling. [#4114, gearnode]
|
162
314
|
|
315
|
+
5.2.9
|
316
|
+
---------
|
317
|
+
|
318
|
+
- Release Rack lock due to a cascade of CVEs. [#4566]
|
319
|
+
Pro-tip: don't lock Rack.
|
320
|
+
|
163
321
|
5.2.8
|
164
322
|
---------
|
165
323
|
|
data/LICENSE
CHANGED
@@ -4,6 +4,6 @@ Sidekiq is an Open Source project licensed under the terms of
|
|
4
4
|
the LGPLv3 license. Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
|
5
5
|
for license text.
|
6
6
|
|
7
|
-
Sidekiq Pro
|
8
|
-
|
9
|
-
|
7
|
+
Sidekiq Pro and Sidekiq Enterprise have a commercial-friendly license.
|
8
|
+
You can find the commercial license in COMM-LICENSE.txt.
|
9
|
+
Please see https://sidekiq.org for purchasing options.
|
data/README.md
CHANGED
@@ -2,11 +2,7 @@ Sidekiq
|
|
2
2
|
==============
|
3
3
|
|
4
4
|
[![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
|
5
|
-
|
6
|
-
[![Test Coverage](https://codeclimate.com/github/mperham/sidekiq/badges/coverage.svg)](https://codeclimate.com/github/mperham/sidekiq/coverage)
|
7
|
-
[![Build Status](https://circleci.com/gh/mperham/sidekiq/tree/master.svg?style=svg)](https://circleci.com/gh/mperham/sidekiq/tree/master)
|
8
|
-
[![Gitter Chat](https://badges.gitter.im/mperham/sidekiq.svg)](https://gitter.im/mperham/sidekiq)
|
9
|
-
|
5
|
+
![Build](https://github.com/mperham/sidekiq/workflows/CI/badge.svg)
|
10
6
|
|
11
7
|
Simple, efficient background processing for Ruby.
|
12
8
|
|
@@ -47,10 +43,10 @@ Getting Started
|
|
47
43
|
-----------------
|
48
44
|
|
49
45
|
See the [Getting Started wiki page](https://github.com/mperham/sidekiq/wiki/Getting-Started) and follow the simple setup process.
|
50
|
-
You can watch [this
|
46
|
+
You can watch [this YouTube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
|
51
47
|
Sidekiq and see its features in action. Here's the Web UI:
|
52
48
|
|
53
|
-
![Web UI](https://github.com/mperham/sidekiq/raw/
|
49
|
+
![Web UI](https://github.com/mperham/sidekiq/raw/main/examples/web-ui.png)
|
54
50
|
|
55
51
|
|
56
52
|
Want to Upgrade?
|
@@ -88,10 +84,10 @@ See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
|
|
88
84
|
License
|
89
85
|
-----------------
|
90
86
|
|
91
|
-
Please see [LICENSE](https://github.com/mperham/sidekiq/blob/
|
87
|
+
Please see [LICENSE](https://github.com/mperham/sidekiq/blob/main/LICENSE) for licensing details.
|
92
88
|
|
93
89
|
|
94
90
|
Author
|
95
91
|
-----------------
|
96
92
|
|
97
|
-
Mike Perham, [@
|
93
|
+
Mike Perham, [@getajobmike](https://twitter.com/getajobmike) / [@sidekiq](https://twitter.com/sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
|
data/bin/sidekiq
CHANGED
@@ -31,7 +31,12 @@ begin
|
|
31
31
|
cli.run
|
32
32
|
rescue => e
|
33
33
|
raise e if $DEBUG
|
34
|
-
|
35
|
-
|
34
|
+
if Sidekiq.error_handlers.length == 0
|
35
|
+
STDERR.puts e.message
|
36
|
+
STDERR.puts e.backtrace.join("\n")
|
37
|
+
else
|
38
|
+
cli.handle_exception e
|
39
|
+
end
|
40
|
+
|
36
41
|
exit 1
|
37
42
|
end
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require "rails/generators/named_base"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
module Generators # :nodoc:
|
5
|
+
class JobGenerator < ::Rails::Generators::NamedBase # :nodoc:
|
6
|
+
desc "This generator creates a Sidekiq Job in app/sidekiq and a corresponding test"
|
7
|
+
|
8
|
+
check_class_collision suffix: "Job"
|
9
|
+
|
10
|
+
def self.default_generator_root
|
11
|
+
File.dirname(__FILE__)
|
12
|
+
end
|
13
|
+
|
14
|
+
def create_job_file
|
15
|
+
template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
|
16
|
+
end
|
17
|
+
|
18
|
+
def create_test_file
|
19
|
+
return unless test_framework
|
20
|
+
|
21
|
+
if test_framework == :rspec
|
22
|
+
create_job_spec
|
23
|
+
else
|
24
|
+
create_job_test
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def create_job_spec
|
31
|
+
template_file = File.join(
|
32
|
+
"spec/sidekiq",
|
33
|
+
class_path,
|
34
|
+
"#{file_name}_job_spec.rb"
|
35
|
+
)
|
36
|
+
template "job_spec.rb.erb", template_file
|
37
|
+
end
|
38
|
+
|
39
|
+
def create_job_test
|
40
|
+
template_file = File.join(
|
41
|
+
"test/sidekiq",
|
42
|
+
class_path,
|
43
|
+
"#{file_name}_job_test.rb"
|
44
|
+
)
|
45
|
+
template "job_test.rb.erb", template_file
|
46
|
+
end
|
47
|
+
|
48
|
+
def file_name
|
49
|
+
@_file_name ||= super.sub(/_?job\z/i, "")
|
50
|
+
end
|
51
|
+
|
52
|
+
def test_framework
|
53
|
+
::Rails.application.config.generators.options[:rails][:test_framework]
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
data/lib/sidekiq/api.rb
CHANGED
@@ -8,7 +8,7 @@ require "base64"
|
|
8
8
|
module Sidekiq
|
9
9
|
class Stats
|
10
10
|
def initialize
|
11
|
-
|
11
|
+
fetch_stats_fast!
|
12
12
|
end
|
13
13
|
|
14
14
|
def processed
|
@@ -51,7 +51,8 @@ module Sidekiq
|
|
51
51
|
Sidekiq::Stats::Queues.new.lengths
|
52
52
|
end
|
53
53
|
|
54
|
-
|
54
|
+
# O(1) redis calls
|
55
|
+
def fetch_stats_fast!
|
55
56
|
pipe1_res = Sidekiq.redis { |conn|
|
56
57
|
conn.pipelined do
|
57
58
|
conn.get("stat:processed")
|
@@ -64,6 +65,33 @@ module Sidekiq
|
|
64
65
|
end
|
65
66
|
}
|
66
67
|
|
68
|
+
default_queue_latency = if (entry = pipe1_res[6].first)
|
69
|
+
job = begin
|
70
|
+
Sidekiq.load_json(entry)
|
71
|
+
rescue
|
72
|
+
{}
|
73
|
+
end
|
74
|
+
now = Time.now.to_f
|
75
|
+
thence = job["enqueued_at"] || now
|
76
|
+
now - thence
|
77
|
+
else
|
78
|
+
0
|
79
|
+
end
|
80
|
+
|
81
|
+
@stats = {
|
82
|
+
processed: pipe1_res[0].to_i,
|
83
|
+
failed: pipe1_res[1].to_i,
|
84
|
+
scheduled_size: pipe1_res[2],
|
85
|
+
retry_size: pipe1_res[3],
|
86
|
+
dead_size: pipe1_res[4],
|
87
|
+
processes_size: pipe1_res[5],
|
88
|
+
|
89
|
+
default_queue_latency: default_queue_latency
|
90
|
+
}
|
91
|
+
end
|
92
|
+
|
93
|
+
# O(number of processes + number of queues) redis calls
|
94
|
+
def fetch_stats_slow!
|
67
95
|
processes = Sidekiq.redis { |conn|
|
68
96
|
conn.sscan_each("processes").to_a
|
69
97
|
}
|
@@ -83,30 +111,14 @@ module Sidekiq
|
|
83
111
|
workers_size = pipe2_res[0...s].sum(&:to_i)
|
84
112
|
enqueued = pipe2_res[s..-1].sum(&:to_i)
|
85
113
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
{}
|
91
|
-
end
|
92
|
-
now = Time.now.to_f
|
93
|
-
thence = job["enqueued_at"] || now
|
94
|
-
now - thence
|
95
|
-
else
|
96
|
-
0
|
97
|
-
end
|
98
|
-
@stats = {
|
99
|
-
processed: pipe1_res[0].to_i,
|
100
|
-
failed: pipe1_res[1].to_i,
|
101
|
-
scheduled_size: pipe1_res[2],
|
102
|
-
retry_size: pipe1_res[3],
|
103
|
-
dead_size: pipe1_res[4],
|
104
|
-
processes_size: pipe1_res[5],
|
114
|
+
@stats[:workers_size] = workers_size
|
115
|
+
@stats[:enqueued] = enqueued
|
116
|
+
@stats
|
117
|
+
end
|
105
118
|
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
}
|
119
|
+
def fetch_stats!
|
120
|
+
fetch_stats_fast!
|
121
|
+
fetch_stats_slow!
|
110
122
|
end
|
111
123
|
|
112
124
|
def reset(*stats)
|
@@ -126,7 +138,8 @@ module Sidekiq
|
|
126
138
|
private
|
127
139
|
|
128
140
|
def stat(s)
|
129
|
-
@stats[s]
|
141
|
+
fetch_stats_slow! if @stats[s].nil?
|
142
|
+
@stats[s] || raise(ArgumentError, "Unknown stat #{s}")
|
130
143
|
end
|
131
144
|
|
132
145
|
class Queues
|
@@ -141,13 +154,15 @@ module Sidekiq
|
|
141
154
|
}
|
142
155
|
|
143
156
|
array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
|
144
|
-
|
157
|
+
array_of_arrays.to_h
|
145
158
|
end
|
146
159
|
end
|
147
160
|
end
|
148
161
|
|
149
162
|
class History
|
150
163
|
def initialize(days_previous, start_date = nil)
|
164
|
+
# we only store five years of data in Redis
|
165
|
+
raise ArgumentError if days_previous < 1 || days_previous > (5 * 365)
|
151
166
|
@days_previous = days_previous
|
152
167
|
@start_date = start_date || Time.now.utc.to_date
|
153
168
|
end
|
@@ -255,7 +270,7 @@ module Sidekiq
|
|
255
270
|
break if entries.empty?
|
256
271
|
page += 1
|
257
272
|
entries.each do |entry|
|
258
|
-
yield
|
273
|
+
yield JobRecord.new(entry, @name)
|
259
274
|
end
|
260
275
|
deleted_size = initial_size - size
|
261
276
|
end
|
@@ -265,7 +280,7 @@ module Sidekiq
|
|
265
280
|
# Find the job with the given JID within this queue.
|
266
281
|
#
|
267
282
|
# This is a slow, inefficient operation. Do not use under
|
268
|
-
# normal conditions.
|
283
|
+
# normal conditions.
|
269
284
|
def find_job(jid)
|
270
285
|
detect { |j| j.jid == jid }
|
271
286
|
end
|
@@ -286,9 +301,9 @@ module Sidekiq
|
|
286
301
|
# sorted set.
|
287
302
|
#
|
288
303
|
# The job should be considered immutable but may be
|
289
|
-
# removed from the queue via
|
304
|
+
# removed from the queue via JobRecord#delete.
|
290
305
|
#
|
291
|
-
class
|
306
|
+
class JobRecord
|
292
307
|
attr_reader :item
|
293
308
|
attr_reader :value
|
294
309
|
|
@@ -316,21 +331,23 @@ module Sidekiq
|
|
316
331
|
|
317
332
|
def display_class
|
318
333
|
# Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
|
319
|
-
@klass ||=
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
+
@klass ||= self["display_class"] || begin
|
335
|
+
case klass
|
336
|
+
when /\ASidekiq::Extensions::Delayed/
|
337
|
+
safe_load(args[0], klass) do |target, method, _|
|
338
|
+
"#{target}.#{method}"
|
339
|
+
end
|
340
|
+
when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
|
341
|
+
job_class = @item["wrapped"] || args[0]
|
342
|
+
if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
|
343
|
+
# MailerClass#mailer_method
|
344
|
+
args[0]["arguments"][0..1].join("#")
|
345
|
+
else
|
346
|
+
job_class
|
347
|
+
end
|
348
|
+
else
|
349
|
+
klass
|
350
|
+
end
|
334
351
|
end
|
335
352
|
end
|
336
353
|
|
@@ -443,7 +460,7 @@ module Sidekiq
|
|
443
460
|
end
|
444
461
|
end
|
445
462
|
|
446
|
-
class SortedEntry <
|
463
|
+
class SortedEntry < JobRecord
|
447
464
|
attr_reader :score
|
448
465
|
attr_reader :parent
|
449
466
|
|
@@ -791,19 +808,23 @@ module Sidekiq
|
|
791
808
|
# you'll be happier this way
|
792
809
|
conn.pipelined do
|
793
810
|
procs.each do |key|
|
794
|
-
conn.hmget(key, "info", "busy", "beat", "quiet")
|
811
|
+
conn.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
|
795
812
|
end
|
796
813
|
end
|
797
814
|
}
|
798
815
|
|
799
|
-
result.each do |info, busy, at_s, quiet|
|
816
|
+
result.each do |info, busy, at_s, quiet, rss, rtt|
|
800
817
|
# If a process is stopped between when we query Redis for `procs` and
|
801
818
|
# when we query for `result`, we will have an item in `result` that is
|
802
819
|
# composed of `nil` values.
|
803
820
|
next if info.nil?
|
804
821
|
|
805
822
|
hash = Sidekiq.load_json(info)
|
806
|
-
yield Process.new(hash.merge("busy" => busy.to_i,
|
823
|
+
yield Process.new(hash.merge("busy" => busy.to_i,
|
824
|
+
"beat" => at_s.to_f,
|
825
|
+
"quiet" => quiet,
|
826
|
+
"rss" => rss.to_i,
|
827
|
+
"rtt_us" => rtt.to_i))
|
807
828
|
end
|
808
829
|
end
|
809
830
|
|
@@ -815,6 +836,18 @@ module Sidekiq
|
|
815
836
|
Sidekiq.redis { |conn| conn.scard("processes") }
|
816
837
|
end
|
817
838
|
|
839
|
+
# Total number of threads available to execute jobs.
|
840
|
+
# For Sidekiq Enterprise customers this number (in production) must be
|
841
|
+
# less than or equal to your licensed concurrency.
|
842
|
+
def total_concurrency
|
843
|
+
sum { |x| x["concurrency"].to_i }
|
844
|
+
end
|
845
|
+
|
846
|
+
def total_rss_in_kb
|
847
|
+
sum { |x| x["rss"].to_i }
|
848
|
+
end
|
849
|
+
alias_method :total_rss, :total_rss_in_kb
|
850
|
+
|
818
851
|
# Returns the identity of the current cluster leader or "" if no leader.
|
819
852
|
# This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
|
820
853
|
# or Sidekiq Pro.
|
@@ -864,6 +897,10 @@ module Sidekiq
|
|
864
897
|
self["identity"]
|
865
898
|
end
|
866
899
|
|
900
|
+
def queues
|
901
|
+
self["queues"]
|
902
|
+
end
|
903
|
+
|
867
904
|
def quiet!
|
868
905
|
signal("TSTP")
|
869
906
|
end
|
@@ -894,8 +931,8 @@ module Sidekiq
|
|
894
931
|
end
|
895
932
|
|
896
933
|
##
|
897
|
-
#
|
898
|
-
#
|
934
|
+
# The WorkSet stores the work being done by this Sidekiq cluster.
|
935
|
+
# It tracks the process and thread working on each job.
|
899
936
|
#
|
900
937
|
# WARNING WARNING WARNING
|
901
938
|
#
|
@@ -903,25 +940,26 @@ module Sidekiq
|
|
903
940
|
# If you call #size => 5 and then expect #each to be
|
904
941
|
# called 5 times, you're going to have a bad time.
|
905
942
|
#
|
906
|
-
#
|
907
|
-
#
|
908
|
-
#
|
943
|
+
# works = Sidekiq::WorkSet.new
|
944
|
+
# works.size => 2
|
945
|
+
# works.each do |process_id, thread_id, work|
|
909
946
|
# # process_id is a unique identifier per Sidekiq process
|
910
947
|
# # thread_id is a unique identifier per thread
|
911
948
|
# # work is a Hash which looks like:
|
912
|
-
# # { 'queue' => name, 'run_at' => timestamp, 'payload' =>
|
949
|
+
# # { 'queue' => name, 'run_at' => timestamp, 'payload' => job_hash }
|
913
950
|
# # run_at is an epoch Integer.
|
914
951
|
# end
|
915
952
|
#
|
916
|
-
class
|
953
|
+
class WorkSet
|
917
954
|
include Enumerable
|
918
955
|
|
919
|
-
def each
|
956
|
+
def each(&block)
|
957
|
+
results = []
|
920
958
|
Sidekiq.redis do |conn|
|
921
959
|
procs = conn.sscan_each("processes").to_a
|
922
960
|
procs.sort.each do |key|
|
923
961
|
valid, workers = conn.pipelined {
|
924
|
-
conn.exists(key)
|
962
|
+
conn.exists?(key)
|
925
963
|
conn.hgetall("#{key}:workers")
|
926
964
|
}
|
927
965
|
next unless valid
|
@@ -930,10 +968,12 @@ module Sidekiq
|
|
930
968
|
p = hsh["payload"]
|
931
969
|
# avoid breaking API, this is a side effect of the JSON optimization in #4316
|
932
970
|
hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
|
933
|
-
|
971
|
+
results << [key, tid, hsh]
|
934
972
|
end
|
935
973
|
end
|
936
974
|
end
|
975
|
+
|
976
|
+
results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
|
937
977
|
end
|
938
978
|
|
939
979
|
# Note that #size is only as accurate as Sidekiq's heartbeat,
|
@@ -957,4 +997,8 @@ module Sidekiq
|
|
957
997
|
end
|
958
998
|
end
|
959
999
|
end
|
1000
|
+
# Since "worker" is a nebulous term, we've deprecated the use of this class name.
|
1001
|
+
# Is "worker" a process, a type of job, a thread? Undefined!
|
1002
|
+
# WorkSet better describes the data.
|
1003
|
+
Workers = WorkSet
|
960
1004
|
end
|