sidekiq 6.0.7 → 6.4.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (95) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +160 -2
  3. data/LICENSE +3 -3
  4. data/README.md +5 -9
  5. data/bin/sidekiq +7 -2
  6. data/lib/generators/sidekiq/job_generator.rb +57 -0
  7. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  8. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  9. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  10. data/lib/sidekiq/api.rb +104 -60
  11. data/lib/sidekiq/cli.rb +36 -10
  12. data/lib/sidekiq/client.rb +12 -45
  13. data/lib/sidekiq/delay.rb +2 -0
  14. data/lib/sidekiq/extensions/action_mailer.rb +5 -4
  15. data/lib/sidekiq/extensions/active_record.rb +6 -5
  16. data/lib/sidekiq/extensions/class_methods.rb +7 -6
  17. data/lib/sidekiq/extensions/generic_proxy.rb +5 -3
  18. data/lib/sidekiq/fetch.rb +30 -21
  19. data/lib/sidekiq/job.rb +13 -0
  20. data/lib/sidekiq/job_logger.rb +1 -1
  21. data/lib/sidekiq/job_retry.rb +10 -11
  22. data/lib/sidekiq/job_util.rb +65 -0
  23. data/lib/sidekiq/launcher.rb +79 -21
  24. data/lib/sidekiq/logger.rb +3 -2
  25. data/lib/sidekiq/manager.rb +10 -12
  26. data/lib/sidekiq/middleware/chain.rb +6 -4
  27. data/lib/sidekiq/middleware/current_attributes.rb +57 -0
  28. data/lib/sidekiq/processor.rb +4 -4
  29. data/lib/sidekiq/rails.rb +27 -18
  30. data/lib/sidekiq/redis_connection.rb +14 -13
  31. data/lib/sidekiq/scheduled.rb +51 -16
  32. data/lib/sidekiq/sd_notify.rb +1 -1
  33. data/lib/sidekiq/testing.rb +2 -4
  34. data/lib/sidekiq/util.rb +41 -0
  35. data/lib/sidekiq/version.rb +1 -1
  36. data/lib/sidekiq/web/action.rb +2 -2
  37. data/lib/sidekiq/web/application.rb +21 -12
  38. data/lib/sidekiq/web/csrf_protection.rb +180 -0
  39. data/lib/sidekiq/web/helpers.rb +35 -29
  40. data/lib/sidekiq/web/router.rb +5 -2
  41. data/lib/sidekiq/web.rb +36 -72
  42. data/lib/sidekiq/worker.rb +129 -12
  43. data/lib/sidekiq.rb +12 -3
  44. data/sidekiq.gemspec +11 -4
  45. data/web/assets/images/apple-touch-icon.png +0 -0
  46. data/web/assets/javascripts/application.js +82 -66
  47. data/web/assets/javascripts/dashboard.js +51 -51
  48. data/web/assets/stylesheets/application-dark.css +64 -43
  49. data/web/assets/stylesheets/application-rtl.css +0 -4
  50. data/web/assets/stylesheets/application.css +41 -239
  51. data/web/locales/ar.yml +8 -2
  52. data/web/locales/en.yml +4 -1
  53. data/web/locales/es.yml +18 -2
  54. data/web/locales/fr.yml +8 -1
  55. data/web/locales/ja.yml +3 -0
  56. data/web/locales/lt.yml +1 -1
  57. data/web/locales/pl.yml +4 -4
  58. data/web/locales/ru.yml +4 -0
  59. data/web/views/_footer.erb +1 -1
  60. data/web/views/_job_info.erb +1 -1
  61. data/web/views/_poll_link.erb +2 -5
  62. data/web/views/_summary.erb +7 -7
  63. data/web/views/busy.erb +50 -19
  64. data/web/views/dashboard.erb +22 -14
  65. data/web/views/dead.erb +1 -1
  66. data/web/views/layout.erb +2 -1
  67. data/web/views/morgue.erb +6 -6
  68. data/web/views/queue.erb +11 -11
  69. data/web/views/queues.erb +4 -4
  70. data/web/views/retries.erb +7 -7
  71. data/web/views/retry.erb +1 -1
  72. data/web/views/scheduled.erb +1 -1
  73. metadata +24 -49
  74. data/.circleci/config.yml +0 -60
  75. data/.github/contributing.md +0 -32
  76. data/.github/issue_template.md +0 -11
  77. data/.gitignore +0 -13
  78. data/.standard.yml +0 -20
  79. data/3.0-Upgrade.md +0 -70
  80. data/4.0-Upgrade.md +0 -53
  81. data/5.0-Upgrade.md +0 -56
  82. data/6.0-Upgrade.md +0 -72
  83. data/COMM-LICENSE +0 -97
  84. data/Ent-2.0-Upgrade.md +0 -37
  85. data/Ent-Changes.md +0 -256
  86. data/Gemfile +0 -24
  87. data/Gemfile.lock +0 -208
  88. data/Pro-2.0-Upgrade.md +0 -138
  89. data/Pro-3.0-Upgrade.md +0 -44
  90. data/Pro-4.0-Upgrade.md +0 -35
  91. data/Pro-5.0-Upgrade.md +0 -25
  92. data/Pro-Changes.md +0 -782
  93. data/Rakefile +0 -10
  94. data/code_of_conduct.md +0 -50
  95. data/lib/generators/sidekiq/worker_generator.rb +0 -57
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3dd5d7a0dd8a7d877484df4ae3b9988f15e9cc4bc0bb4da23224be0049fa733e
4
- data.tar.gz: e2b7ddb4c045817837e996356851648bf32108b37fa1abcfb72795a185b4a238
3
+ metadata.gz: 9622b2851203b0c5a80695ab7801ca77e15dc63a641ae79132cda9a2fcbe0cc6
4
+ data.tar.gz: dd943a02d2cf910f51866d02254f3a7845cb159735bd54d223dd7127a1fbd2fa
5
5
  SHA512:
6
- metadata.gz: 5ce3f911b835b38ce212309bcdf81b4789e24a12898a0f2071b9adb9cbcdfa23a33bb7216c2f7c6dcab26d484f79a1271594d3218f63ed041f0027deab5a26f3
7
- data.tar.gz: 685a6890a43c990b2fd9709c49f9c82c96a487cdcf6fc0607ab15b9dbb47d52c9f797ec5094d3f92ec35c5737e661f966a6b81006ee4b7d99a7f31285180b3db
6
+ metadata.gz: 65bcd542866d8699ecf5958d81a15fd322cd1c51dfc1dbc6a8b15402b70862510c134e2b0ed9f9dbcdbb4dea3a59c1d12878ff926145e503079a59896f279ff3
7
+ data.tar.gz: 36143e85dc7fd4611f8a43fe39788dc590725ac63a827fbc174916da2d59fdadb3fb64fa0d819c4ef90694f6f292ea15f7fb58fb29f368c0a06c5b90f1b4bca7
data/Changes.md CHANGED
@@ -1,6 +1,158 @@
1
1
  # Sidekiq Changes
2
2
 
3
- [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
3
+ [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
4
+
5
+ 6.4.0
6
+ ---------
7
+
8
+ - **SECURITY**: Validate input to avoid possible DoS in Web UI.
9
+ - Add **strict argument checking** [#5071]
10
+ Sidekiq will now log a warning if JSON-unsafe arguments are passed to `perform_async`.
11
+ Add `Sidekiq.strict_args!(false)` to your initializer to disable this warning.
12
+ This warning will switch to an exception in Sidekiq 7.0.
13
+ - Note that Delayed Extensions will be removed in Sidekiq 7.0 [#5076]
14
+ - Add `perform_{inline,sync}` in Sidekiq::Job to run a job synchronously [#5061, hasan-ally]
15
+ ```ruby
16
+ SomeJob.perform_async(args...)
17
+ SomeJob.perform_sync(args...)
18
+ SomeJob.perform_inline(args...)
19
+ ```
20
+ You can also dynamically redirect a job to run synchronously:
21
+ ```ruby
22
+ SomeJob.set("sync": true).perform_async(args...) # will run via perform_inline
23
+ ```
24
+ - Replace Sidekiq::Worker `app/workers` generator with Sidekiq::Job `app/sidekiq` generator [#5055]
25
+ ```
26
+ bin/rails generate sidekiq:job ProcessOrderJob
27
+ ```
28
+ - Fix job retries losing CurrentAttributes [#5090]
29
+ - Tweak shutdown to give long-running threads time to cleanup [#5095]
30
+ - Add keyword arguments support in extensions
31
+
32
+ 6.3.1
33
+ ---------
34
+
35
+ - Fix keyword arguments error with CurrentAttributes on Ruby 3.0 [#5048]
36
+
37
+ 6.3.0
38
+ ---------
39
+
40
+ - **BREAK**: The Web UI has been refactored to remove jQuery. Any UI extensions
41
+ which use jQuery will break.
42
+ - **FEATURE**: Sidekiq.logger has been enhanced so any `Rails.logger`
43
+ output in jobs now shows up in the Sidekiq console. Remove any logger
44
+ hacks in your initializer and see if it Just Works™ now. [#5021]
45
+ - **FEATURE**: Add `Sidekiq::Job` alias for `Sidekiq::Worker`, to better
46
+ reflect industry standard terminology. You can now do this:
47
+ ```ruby
48
+ class MyJob
49
+ include Sidekiq::Job
50
+ sidekiq_options ...
51
+ def perform(args)
52
+ end
53
+ end
54
+ ```
55
+ - **FEATURE**: Support for serializing ActiveSupport::CurrentAttributes into each job. [#4982]
56
+ ```ruby
57
+ # config/initializers/sidekiq.rb
58
+ require "sidekiq/middleware/current_attributes"
59
+ Sidekiq::CurrentAttributes.persist(Myapp::Current) # Your AS::CurrentAttributes singleton
60
+ ```
61
+ - **FEATURE**: Add `Sidekiq::Worker.perform_bulk` for enqueuing jobs in bulk,
62
+ similar to `Sidekiq::Client.push_bulk` [#5042]
63
+ ```ruby
64
+ MyJob.perform_bulk([[1], [2], [3]])
65
+ ```
66
+ - Implement `queue_as`, `wait` and `wait_until` for ActiveJob compatibility [#5003]
67
+ - Scheduler now uses Lua to reduce Redis load and network roundtrips [#5044]
68
+ - Retry Redis operation if we get an `UNBLOCKED` Redis error [#4985]
69
+ - Run existing signal traps, if any, before running Sidekiq's trap [#4991]
70
+ - Fix fetch bug when using weighted queues which caused Sidekiq to stop
71
+ processing queues randomly [#5031]
72
+
73
+ 6.2.2
74
+ ---------
75
+
76
+ - Reduce retry jitter, add jitter to `sidekiq_retry_in` values [#4957]
77
+ - Minimize scheduler load on Redis at scale [#4882]
78
+ - Improve logging of delay jobs [#4904, BuonOno]
79
+ - Minor CSS improvements for buttons and tables, design PRs always welcome!
80
+ - Tweak Web UI `Cache-Control` header [#4966]
81
+ - Rename internal API class `Sidekiq::Job` to `Sidekiq::JobRecord` [#4955]
82
+
83
+ 6.2.1
84
+ ---------
85
+
86
+ - Update RTT warning logic to handle transient RTT spikes [#4851]
87
+ - Fix very low priority CVE on unescaped queue name [#4852]
88
+ - Add note about sessions and Rails apps in API mode
89
+
90
+ 6.2.0
91
+ ---------
92
+
93
+ - Store Redis RTT and log if poor [#4824]
94
+ - Add process/thread stats to Busy page [#4806]
95
+ - Improve Web UI on mobile devices [#4840]
96
+ - **Refactor Web UI session usage** [#4804]
97
+ Numerous people have hit "Forbidden" errors and struggled with Sidekiq's
98
+ Web UI session requirement. If you have code in your initializer for
99
+ Web sessions, it's quite possible it will need to be removed. Here's
100
+ an overview:
101
+ ```
102
+ Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app,
103
+ make sure you mount Sidekiq::Web *inside* your routes in `config/routes.rb` so
104
+ Sidekiq can reuse the Rails session:
105
+
106
+ Rails.application.routes.draw do
107
+ mount Sidekiq::Web => "/sidekiq"
108
+ ....
109
+ end
110
+
111
+ If this is a bare Rack app, use a session middleware before Sidekiq::Web:
112
+
113
+ # first, use IRB to create a shared secret key for sessions and commit it
114
+ require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
115
+
116
+ # now, update your Rack app to include the secret with a session cookie middleware
117
+ use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
118
+ run Sidekiq::Web
119
+
120
+ If this is a Rails app in API mode, you need to enable sessions.
121
+
122
+ https://guides.rubyonrails.org/api_app.html#using-session-middlewares
123
+ ```
124
+
125
+ 6.1.3
126
+ ---------
127
+
128
+ - Warn if Redis is configured to evict data under memory pressure [#4752]
129
+ - Add process RSS on the Busy page [#4717]
130
+
131
+ 6.1.2
132
+ ---------
133
+
134
+ - Improve readability in dark mode Web UI [#4674]
135
+ - Fix Web UI crash with corrupt session [#4672]
136
+ - Allow middleware to yield arguments [#4673, @eugeneius]
137
+ - Migrate CI from CircleCI to GitHub Actions [#4677]
138
+
139
+ 6.1.1
140
+ ---------
141
+
142
+ - Jobs are now sorted by age in the Busy Workers table. [#4641]
143
+ - Fix "check all" JS logic in Web UI [#4619]
144
+
145
+ 6.1.0
146
+ ---------
147
+
148
+ - Web UI - Dark Mode fixes [#4543, natematykiewicz]
149
+ - Ensure `Rack::ContentLength` is loaded as middleware for correct Web UI responses [#4541]
150
+ - Avoid exception dumping SSL store in Redis connection logging [#4532]
151
+ - Better error messages in Sidekiq::Client [#4549]
152
+ - Remove rack-protection, reimplement CSRF protection [#4588]
153
+ - Require redis-rb 4.2 [#4591]
154
+ - Update to jquery 1.12.4 [#4593]
155
+ - Refactor internal fetch logic and API [#4602]
4
156
 
5
157
  6.0.7
6
158
  ---------
@@ -124,7 +276,7 @@ assert_equal 1, Sidekiq::Extensions::DelayedMailer.jobs_for(FooMailer).size
124
276
 
125
277
  This release has major breaking changes. Read and test carefully in production.
126
278
 
127
- - With Rails 6.0.1+, ActiveJobs can now use `sidekiq_options` directly to configure Sidekiq
279
+ - With Rails 6.0.2+, ActiveJobs can now use `sidekiq_options` directly to configure Sidekiq
128
280
  features/internals like the retry subsystem. [#4213, pirj]
129
281
  ```ruby
130
282
  class MyJob < ActiveJob::Base
@@ -160,6 +312,12 @@ See the [Logging wiki page](https://github.com/mperham/sidekiq/wiki/Logging) for
160
312
  - Integrate the StandardRB code formatter to ensure consistent code
161
313
  styling. [#4114, gearnode]
162
314
 
315
+ 5.2.9
316
+ ---------
317
+
318
+ - Release Rack lock due to a cascade of CVEs. [#4566]
319
+ Pro-tip: don't lock Rack.
320
+
163
321
  5.2.8
164
322
  ---------
165
323
 
data/LICENSE CHANGED
@@ -4,6 +4,6 @@ Sidekiq is an Open Source project licensed under the terms of
4
4
  the LGPLv3 license. Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
5
5
  for license text.
6
6
 
7
- Sidekiq Pro has a commercial-friendly license allowing private forks
8
- and modifications of Sidekiq. Please see https://sidekiq.org/products/pro.html for
9
- more detail. You can find the commercial license terms in COMM-LICENSE.
7
+ Sidekiq Pro and Sidekiq Enterprise have a commercial-friendly license.
8
+ You can find the commercial license in COMM-LICENSE.txt.
9
+ Please see https://sidekiq.org for purchasing options.
data/README.md CHANGED
@@ -2,11 +2,7 @@ Sidekiq
2
2
  ==============
3
3
 
4
4
  [![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
5
- [![Code Climate](https://codeclimate.com/github/mperham/sidekiq.svg)](https://codeclimate.com/github/mperham/sidekiq)
6
- [![Test Coverage](https://codeclimate.com/github/mperham/sidekiq/badges/coverage.svg)](https://codeclimate.com/github/mperham/sidekiq/coverage)
7
- [![Build Status](https://circleci.com/gh/mperham/sidekiq/tree/master.svg?style=svg)](https://circleci.com/gh/mperham/sidekiq/tree/master)
8
- [![Gitter Chat](https://badges.gitter.im/mperham/sidekiq.svg)](https://gitter.im/mperham/sidekiq)
9
-
5
+ ![Build](https://github.com/mperham/sidekiq/workflows/CI/badge.svg)
10
6
 
11
7
  Simple, efficient background processing for Ruby.
12
8
 
@@ -47,10 +43,10 @@ Getting Started
47
43
  -----------------
48
44
 
49
45
  See the [Getting Started wiki page](https://github.com/mperham/sidekiq/wiki/Getting-Started) and follow the simple setup process.
50
- You can watch [this Youtube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
46
+ You can watch [this YouTube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
51
47
  Sidekiq and see its features in action. Here's the Web UI:
52
48
 
53
- ![Web UI](https://github.com/mperham/sidekiq/raw/master/examples/web-ui.png)
49
+ ![Web UI](https://github.com/mperham/sidekiq/raw/main/examples/web-ui.png)
54
50
 
55
51
 
56
52
  Want to Upgrade?
@@ -88,10 +84,10 @@ See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
88
84
  License
89
85
  -----------------
90
86
 
91
- Please see [LICENSE](https://github.com/mperham/sidekiq/blob/master/LICENSE) for licensing details.
87
+ Please see [LICENSE](https://github.com/mperham/sidekiq/blob/main/LICENSE) for licensing details.
92
88
 
93
89
 
94
90
  Author
95
91
  -----------------
96
92
 
97
- Mike Perham, [@mperham@mastodon.xyz](https://mastodon.xyz/@mperham) / [@sidekiq](https://twitter.com/sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
93
+ Mike Perham, [@getajobmike](https://twitter.com/getajobmike) / [@sidekiq](https://twitter.com/sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
data/bin/sidekiq CHANGED
@@ -31,7 +31,12 @@ begin
31
31
  cli.run
32
32
  rescue => e
33
33
  raise e if $DEBUG
34
- STDERR.puts e.message
35
- STDERR.puts e.backtrace.join("\n")
34
+ if Sidekiq.error_handlers.length == 0
35
+ STDERR.puts e.message
36
+ STDERR.puts e.backtrace.join("\n")
37
+ else
38
+ cli.handle_exception e
39
+ end
40
+
36
41
  exit 1
37
42
  end
@@ -0,0 +1,57 @@
1
+ require "rails/generators/named_base"
2
+
3
+ module Sidekiq
4
+ module Generators # :nodoc:
5
+ class JobGenerator < ::Rails::Generators::NamedBase # :nodoc:
6
+ desc "This generator creates a Sidekiq Job in app/sidekiq and a corresponding test"
7
+
8
+ check_class_collision suffix: "Job"
9
+
10
+ def self.default_generator_root
11
+ File.dirname(__FILE__)
12
+ end
13
+
14
+ def create_job_file
15
+ template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
16
+ end
17
+
18
+ def create_test_file
19
+ return unless test_framework
20
+
21
+ if test_framework == :rspec
22
+ create_job_spec
23
+ else
24
+ create_job_test
25
+ end
26
+ end
27
+
28
+ private
29
+
30
+ def create_job_spec
31
+ template_file = File.join(
32
+ "spec/sidekiq",
33
+ class_path,
34
+ "#{file_name}_job_spec.rb"
35
+ )
36
+ template "job_spec.rb.erb", template_file
37
+ end
38
+
39
+ def create_job_test
40
+ template_file = File.join(
41
+ "test/sidekiq",
42
+ class_path,
43
+ "#{file_name}_job_test.rb"
44
+ )
45
+ template "job_test.rb.erb", template_file
46
+ end
47
+
48
+ def file_name
49
+ @_file_name ||= super.sub(/_?job\z/i, "")
50
+ end
51
+
52
+ def test_framework
53
+ ::Rails.application.config.generators.options[:rails][:test_framework]
54
+ end
55
+ end
56
+ end
57
+ end
@@ -1,6 +1,6 @@
1
1
  <% module_namespacing do -%>
2
- class <%= class_name %>Worker
3
- include Sidekiq::Worker
2
+ class <%= class_name %>Job
3
+ include Sidekiq::Job
4
4
 
5
5
  def perform(*args)
6
6
  # Do something
@@ -1,6 +1,6 @@
1
1
  require 'rails_helper'
2
2
  <% module_namespacing do -%>
3
- RSpec.describe <%= class_name %>Worker, type: :worker do
3
+ RSpec.describe <%= class_name %>Job, type: :job do
4
4
  pending "add some examples to (or delete) #{__FILE__}"
5
5
  end
6
6
  <% end -%>
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
  <% module_namespacing do -%>
3
- class <%= class_name %>WorkerTest < Minitest::Test
3
+ class <%= class_name %>JobTest < Minitest::Test
4
4
  def test_example
5
5
  skip "add some examples to (or delete) #{__FILE__}"
6
6
  end
data/lib/sidekiq/api.rb CHANGED
@@ -8,7 +8,7 @@ require "base64"
8
8
  module Sidekiq
9
9
  class Stats
10
10
  def initialize
11
- fetch_stats!
11
+ fetch_stats_fast!
12
12
  end
13
13
 
14
14
  def processed
@@ -51,7 +51,8 @@ module Sidekiq
51
51
  Sidekiq::Stats::Queues.new.lengths
52
52
  end
53
53
 
54
- def fetch_stats!
54
+ # O(1) redis calls
55
+ def fetch_stats_fast!
55
56
  pipe1_res = Sidekiq.redis { |conn|
56
57
  conn.pipelined do
57
58
  conn.get("stat:processed")
@@ -64,6 +65,33 @@ module Sidekiq
64
65
  end
65
66
  }
66
67
 
68
+ default_queue_latency = if (entry = pipe1_res[6].first)
69
+ job = begin
70
+ Sidekiq.load_json(entry)
71
+ rescue
72
+ {}
73
+ end
74
+ now = Time.now.to_f
75
+ thence = job["enqueued_at"] || now
76
+ now - thence
77
+ else
78
+ 0
79
+ end
80
+
81
+ @stats = {
82
+ processed: pipe1_res[0].to_i,
83
+ failed: pipe1_res[1].to_i,
84
+ scheduled_size: pipe1_res[2],
85
+ retry_size: pipe1_res[3],
86
+ dead_size: pipe1_res[4],
87
+ processes_size: pipe1_res[5],
88
+
89
+ default_queue_latency: default_queue_latency
90
+ }
91
+ end
92
+
93
+ # O(number of processes + number of queues) redis calls
94
+ def fetch_stats_slow!
67
95
  processes = Sidekiq.redis { |conn|
68
96
  conn.sscan_each("processes").to_a
69
97
  }
@@ -83,30 +111,14 @@ module Sidekiq
83
111
  workers_size = pipe2_res[0...s].sum(&:to_i)
84
112
  enqueued = pipe2_res[s..-1].sum(&:to_i)
85
113
 
86
- default_queue_latency = if (entry = pipe1_res[6].first)
87
- job = begin
88
- Sidekiq.load_json(entry)
89
- rescue
90
- {}
91
- end
92
- now = Time.now.to_f
93
- thence = job["enqueued_at"] || now
94
- now - thence
95
- else
96
- 0
97
- end
98
- @stats = {
99
- processed: pipe1_res[0].to_i,
100
- failed: pipe1_res[1].to_i,
101
- scheduled_size: pipe1_res[2],
102
- retry_size: pipe1_res[3],
103
- dead_size: pipe1_res[4],
104
- processes_size: pipe1_res[5],
114
+ @stats[:workers_size] = workers_size
115
+ @stats[:enqueued] = enqueued
116
+ @stats
117
+ end
105
118
 
106
- default_queue_latency: default_queue_latency,
107
- workers_size: workers_size,
108
- enqueued: enqueued
109
- }
119
+ def fetch_stats!
120
+ fetch_stats_fast!
121
+ fetch_stats_slow!
110
122
  end
111
123
 
112
124
  def reset(*stats)
@@ -126,7 +138,8 @@ module Sidekiq
126
138
  private
127
139
 
128
140
  def stat(s)
129
- @stats[s]
141
+ fetch_stats_slow! if @stats[s].nil?
142
+ @stats[s] || raise(ArgumentError, "Unknown stat #{s}")
130
143
  end
131
144
 
132
145
  class Queues
@@ -141,13 +154,15 @@ module Sidekiq
141
154
  }
142
155
 
143
156
  array_of_arrays = queues.zip(lengths).sort_by { |_, size| -size }
144
- Hash[array_of_arrays]
157
+ array_of_arrays.to_h
145
158
  end
146
159
  end
147
160
  end
148
161
 
149
162
  class History
150
163
  def initialize(days_previous, start_date = nil)
164
+ # we only store five years of data in Redis
165
+ raise ArgumentError if days_previous < 1 || days_previous > (5 * 365)
151
166
  @days_previous = days_previous
152
167
  @start_date = start_date || Time.now.utc.to_date
153
168
  end
@@ -255,7 +270,7 @@ module Sidekiq
255
270
  break if entries.empty?
256
271
  page += 1
257
272
  entries.each do |entry|
258
- yield Job.new(entry, @name)
273
+ yield JobRecord.new(entry, @name)
259
274
  end
260
275
  deleted_size = initial_size - size
261
276
  end
@@ -265,7 +280,7 @@ module Sidekiq
265
280
  # Find the job with the given JID within this queue.
266
281
  #
267
282
  # This is a slow, inefficient operation. Do not use under
268
- # normal conditions. Sidekiq Pro contains a faster version.
283
+ # normal conditions.
269
284
  def find_job(jid)
270
285
  detect { |j| j.jid == jid }
271
286
  end
@@ -286,9 +301,9 @@ module Sidekiq
286
301
  # sorted set.
287
302
  #
288
303
  # The job should be considered immutable but may be
289
- # removed from the queue via Job#delete.
304
+ # removed from the queue via JobRecord#delete.
290
305
  #
291
- class Job
306
+ class JobRecord
292
307
  attr_reader :item
293
308
  attr_reader :value
294
309
 
@@ -316,21 +331,23 @@ module Sidekiq
316
331
 
317
332
  def display_class
318
333
  # Unwrap known wrappers so they show up in a human-friendly manner in the Web UI
319
- @klass ||= case klass
320
- when /\ASidekiq::Extensions::Delayed/
321
- safe_load(args[0], klass) do |target, method, _|
322
- "#{target}.#{method}"
323
- end
324
- when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
325
- job_class = @item["wrapped"] || args[0]
326
- if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
327
- # MailerClass#mailer_method
328
- args[0]["arguments"][0..1].join("#")
329
- else
330
- job_class
331
- end
332
- else
333
- klass
334
+ @klass ||= self["display_class"] || begin
335
+ case klass
336
+ when /\ASidekiq::Extensions::Delayed/
337
+ safe_load(args[0], klass) do |target, method, _|
338
+ "#{target}.#{method}"
339
+ end
340
+ when "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
341
+ job_class = @item["wrapped"] || args[0]
342
+ if job_class == "ActionMailer::DeliveryJob" || job_class == "ActionMailer::MailDeliveryJob"
343
+ # MailerClass#mailer_method
344
+ args[0]["arguments"][0..1].join("#")
345
+ else
346
+ job_class
347
+ end
348
+ else
349
+ klass
350
+ end
334
351
  end
335
352
  end
336
353
 
@@ -443,7 +460,7 @@ module Sidekiq
443
460
  end
444
461
  end
445
462
 
446
- class SortedEntry < Job
463
+ class SortedEntry < JobRecord
447
464
  attr_reader :score
448
465
  attr_reader :parent
449
466
 
@@ -791,19 +808,23 @@ module Sidekiq
791
808
  # you'll be happier this way
792
809
  conn.pipelined do
793
810
  procs.each do |key|
794
- conn.hmget(key, "info", "busy", "beat", "quiet")
811
+ conn.hmget(key, "info", "busy", "beat", "quiet", "rss", "rtt_us")
795
812
  end
796
813
  end
797
814
  }
798
815
 
799
- result.each do |info, busy, at_s, quiet|
816
+ result.each do |info, busy, at_s, quiet, rss, rtt|
800
817
  # If a process is stopped between when we query Redis for `procs` and
801
818
  # when we query for `result`, we will have an item in `result` that is
802
819
  # composed of `nil` values.
803
820
  next if info.nil?
804
821
 
805
822
  hash = Sidekiq.load_json(info)
806
- yield Process.new(hash.merge("busy" => busy.to_i, "beat" => at_s.to_f, "quiet" => quiet))
823
+ yield Process.new(hash.merge("busy" => busy.to_i,
824
+ "beat" => at_s.to_f,
825
+ "quiet" => quiet,
826
+ "rss" => rss.to_i,
827
+ "rtt_us" => rtt.to_i))
807
828
  end
808
829
  end
809
830
 
@@ -815,6 +836,18 @@ module Sidekiq
815
836
  Sidekiq.redis { |conn| conn.scard("processes") }
816
837
  end
817
838
 
839
+ # Total number of threads available to execute jobs.
840
+ # For Sidekiq Enterprise customers this number (in production) must be
841
+ # less than or equal to your licensed concurrency.
842
+ def total_concurrency
843
+ sum { |x| x["concurrency"].to_i }
844
+ end
845
+
846
+ def total_rss_in_kb
847
+ sum { |x| x["rss"].to_i }
848
+ end
849
+ alias_method :total_rss, :total_rss_in_kb
850
+
818
851
  # Returns the identity of the current cluster leader or "" if no leader.
819
852
  # This is a Sidekiq Enterprise feature, will always return "" in Sidekiq
820
853
  # or Sidekiq Pro.
@@ -864,6 +897,10 @@ module Sidekiq
864
897
  self["identity"]
865
898
  end
866
899
 
900
+ def queues
901
+ self["queues"]
902
+ end
903
+
867
904
  def quiet!
868
905
  signal("TSTP")
869
906
  end
@@ -894,8 +931,8 @@ module Sidekiq
894
931
  end
895
932
 
896
933
  ##
897
- # A worker is a thread that is currently processing a job.
898
- # Programmatic access to the current active worker set.
934
+ # The WorkSet stores the work being done by this Sidekiq cluster.
935
+ # It tracks the process and thread working on each job.
899
936
  #
900
937
  # WARNING WARNING WARNING
901
938
  #
@@ -903,25 +940,26 @@ module Sidekiq
903
940
  # If you call #size => 5 and then expect #each to be
904
941
  # called 5 times, you're going to have a bad time.
905
942
  #
906
- # workers = Sidekiq::Workers.new
907
- # workers.size => 2
908
- # workers.each do |process_id, thread_id, work|
943
+ # works = Sidekiq::WorkSet.new
944
+ # works.size => 2
945
+ # works.each do |process_id, thread_id, work|
909
946
  # # process_id is a unique identifier per Sidekiq process
910
947
  # # thread_id is a unique identifier per thread
911
948
  # # work is a Hash which looks like:
912
- # # { 'queue' => name, 'run_at' => timestamp, 'payload' => msg }
949
+ # # { 'queue' => name, 'run_at' => timestamp, 'payload' => job_hash }
913
950
  # # run_at is an epoch Integer.
914
951
  # end
915
952
  #
916
- class Workers
953
+ class WorkSet
917
954
  include Enumerable
918
955
 
919
- def each
956
+ def each(&block)
957
+ results = []
920
958
  Sidekiq.redis do |conn|
921
959
  procs = conn.sscan_each("processes").to_a
922
960
  procs.sort.each do |key|
923
961
  valid, workers = conn.pipelined {
924
- conn.exists(key)
962
+ conn.exists?(key)
925
963
  conn.hgetall("#{key}:workers")
926
964
  }
927
965
  next unless valid
@@ -930,10 +968,12 @@ module Sidekiq
930
968
  p = hsh["payload"]
931
969
  # avoid breaking API, this is a side effect of the JSON optimization in #4316
932
970
  hsh["payload"] = Sidekiq.load_json(p) if p.is_a?(String)
933
- yield key, tid, hsh
971
+ results << [key, tid, hsh]
934
972
  end
935
973
  end
936
974
  end
975
+
976
+ results.sort_by { |(_, _, hsh)| hsh["run_at"] }.each(&block)
937
977
  end
938
978
 
939
979
  # Note that #size is only as accurate as Sidekiq's heartbeat,
@@ -957,4 +997,8 @@ module Sidekiq
957
997
  end
958
998
  end
959
999
  end
1000
+ # Since "worker" is a nebulous term, we've deprecated the use of this class name.
1001
+ # Is "worker" a process, a type of job, a thread? Undefined!
1002
+ # WorkSet better describes the data.
1003
+ Workers = WorkSet
960
1004
  end