sidekiq 5.2.6 → 7.1.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +537 -8
- data/LICENSE.txt +9 -0
- data/README.md +47 -50
- data/bin/sidekiq +22 -3
- data/bin/sidekiqload +213 -115
- data/bin/sidekiqmon +11 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +556 -351
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +203 -226
- data/lib/sidekiq/client.rb +121 -101
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +274 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +49 -42
- data/lib/sidekiq/job.rb +374 -0
- data/lib/sidekiq/job_logger.rb +33 -7
- data/lib/sidekiq/job_retry.rb +131 -108
- data/lib/sidekiq/job_util.rb +105 -0
- data/lib/sidekiq/launcher.rb +203 -105
- data/lib/sidekiq/logger.rb +131 -0
- data/lib/sidekiq/manager.rb +43 -46
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +113 -56
- data/lib/sidekiq/middleware/current_attributes.rb +56 -0
- data/lib/sidekiq/middleware/i18n.rb +7 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +146 -0
- data/lib/sidekiq/paginator.rb +28 -16
- data/lib/sidekiq/processor.rb +108 -107
- data/lib/sidekiq/rails.rb +49 -38
- data/lib/sidekiq/redis_client_adapter.rb +96 -0
- data/lib/sidekiq/redis_connection.rb +38 -107
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +111 -49
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +6 -5
- data/lib/sidekiq/testing.rb +66 -84
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +3 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +123 -79
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +137 -106
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +56 -107
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +92 -182
- data/sidekiq.gemspec +25 -16
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +130 -61
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +166 -0
- data/web/assets/javascripts/dashboard.js +36 -292
- data/web/assets/javascripts/metrics.js +264 -0
- data/web/assets/stylesheets/application-dark.css +147 -0
- data/web/assets/stylesheets/application-rtl.css +2 -95
- data/web/assets/stylesheets/application.css +102 -522
- data/web/locales/ar.yml +71 -65
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -53
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +84 -66
- data/web/locales/es.yml +70 -54
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +83 -62
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +75 -64
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +83 -0
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +63 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +68 -63
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +43 -16
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +6 -3
- data/web/views/_job_info.erb +21 -4
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +1 -1
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +3 -6
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +75 -25
- data/web/views/dashboard.erb +58 -18
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +3 -1
- data/web/views/metrics.erb +82 -0
- data/web/views/metrics_for_job.erb +68 -0
- data/web/views/morgue.erb +14 -15
- data/web/views/queue.erb +33 -24
- data/web/views/queues.erb +13 -3
- data/web/views/retries.erb +16 -17
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +17 -15
- metadata +69 -69
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -15
- data/.travis.yml +0 -11
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/COMM-LICENSE +0 -97
- data/Ent-Changes.md +0 -238
- data/Gemfile +0 -23
- data/LICENSE +0 -9
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-Changes.md +0 -759
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -20
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/ctl.rb +0 -221
- data/lib/sidekiq/delay.rb +0 -42
- data/lib/sidekiq/exception_handler.rb +0 -29
- data/lib/sidekiq/extensions/action_mailer.rb +0 -57
- data/lib/sidekiq/extensions/active_record.rb +0 -40
- data/lib/sidekiq/extensions/class_methods.rb +0 -40
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -31
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
- data/lib/sidekiq/util.rb +0 -66
- data/lib/sidekiq/worker.rb +0 -220
data/README.md
CHANGED
@@ -2,10 +2,7 @@ Sidekiq
|
|
2
2
|
==============
|
3
3
|
|
4
4
|
[![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
|
5
|
-
|
6
|
-
[![Build Status](https://travis-ci.org/mperham/sidekiq.svg)](https://travis-ci.org/mperham/sidekiq)
|
7
|
-
[![Gitter Chat](https://badges.gitter.im/mperham/sidekiq.svg)](https://gitter.im/mperham/sidekiq)
|
8
|
-
|
5
|
+
![Build](https://github.com/sidekiq/sidekiq/workflows/CI/badge.svg)
|
9
6
|
|
10
7
|
Simple, efficient background processing for Ruby.
|
11
8
|
|
@@ -13,97 +10,97 @@ Sidekiq uses threads to handle many jobs at the same time in the
|
|
13
10
|
same process. It does not require Rails but will integrate tightly with
|
14
11
|
Rails to make background processing dead simple.
|
15
12
|
|
16
|
-
Sidekiq is compatible with Resque. It uses the exact same
|
17
|
-
message format as Resque so it can integrate into an existing Resque processing farm.
|
18
|
-
You can have Sidekiq and Resque run side-by-side at the same time and
|
19
|
-
use the Resque client to enqueue jobs in Redis to be processed by Sidekiq.
|
20
|
-
|
21
|
-
Performance
|
22
|
-
---------------
|
23
|
-
|
24
|
-
Version | Latency | Garbage created for 10,000 jobs | Time to process 100,000 jobs | Throughput
|
25
|
-
-----------------|------|---------|---------|------------------------
|
26
|
-
Sidekiq 4.0.0 | 10ms | 151 MB | 22 sec | **4500 jobs/sec**
|
27
|
-
Sidekiq 3.5.1 | 22ms | 1257 MB | 125 sec | 800 jobs/sec
|
28
|
-
Resque 1.25.2 | - | - | 420 sec | 240 jobs/sec
|
29
|
-
DelayedJob 4.1.1 | - | - | 465 sec | 215 jobs/sec
|
30
|
-
|
31
|
-
<small>This benchmark can be found in `bin/sidekiqload`.</small>
|
32
13
|
|
33
14
|
Requirements
|
34
15
|
-----------------
|
35
16
|
|
36
|
-
|
37
|
-
|
38
|
-
All Rails releases >= 4.0 are officially supported.
|
17
|
+
- Redis: 6.2+
|
18
|
+
- Ruby: MRI 2.7+ or JRuby 9.3+.
|
39
19
|
|
40
|
-
|
41
|
-
installations with thousands of worker threads.
|
20
|
+
Sidekiq 7.0 supports Rails 6.0+ but does not require it.
|
42
21
|
|
43
22
|
|
44
23
|
Installation
|
45
24
|
-----------------
|
46
25
|
|
47
|
-
|
26
|
+
bundle add sidekiq
|
48
27
|
|
49
28
|
|
50
29
|
Getting Started
|
51
30
|
-----------------
|
52
31
|
|
53
|
-
See the [Getting Started wiki page](https://github.com/
|
54
|
-
You can watch [this
|
32
|
+
See the [Getting Started wiki page](https://github.com/sidekiq/sidekiq/wiki/Getting-Started) and follow the simple setup process.
|
33
|
+
You can watch [this YouTube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
|
55
34
|
Sidekiq and see its features in action. Here's the Web UI:
|
56
35
|
|
57
|
-
![Web UI](https://github.com/
|
36
|
+
![Web UI](https://github.com/sidekiq/sidekiq/raw/main/examples/web-ui.png)
|
58
37
|
|
38
|
+
Performance
|
39
|
+
---------------
|
40
|
+
|
41
|
+
The benchmark in `bin/sidekiqload` creates 500,000 no-op jobs and drains them as fast as possible, assuming a fixed Redis network latency of 1ms.
|
42
|
+
This requires a lot of Redis network I/O and JSON parsing.
|
43
|
+
This benchmark is IO-bound so we increase the concurrency to 25.
|
44
|
+
If your application is sending lots of emails or performing other network-intensive work, you could see a similar benefit but be careful not to saturate the CPU.
|
45
|
+
|
46
|
+
Version | Time to process 500k jobs | Throughput (jobs/sec) | Ruby | Concurrency | Job Type
|
47
|
+
-----------------|------|---------|---------|------------------------|---
|
48
|
+
Sidekiq 7.0.3 | 21.3 sec| 23,500 | 3.2.0+yjit | 30 | Sidekiq::Job
|
49
|
+
Sidekiq 7.0.3 | 33.8 sec| 14,700 | 3.2.0+yjit | 30 | ActiveJob 7.0.4
|
50
|
+
Sidekiq 7.0.3 | 23.5 sec| 21,300 | 3.2.0 | 30 | Sidekiq::Job
|
51
|
+
Sidekiq 7.0.3 | 46.5 sec| 10,700 | 3.2.0 | 30 | ActiveJob 7.0.4
|
52
|
+
Sidekiq 7.0.3 | 23.0 sec| 21,700 | 2.7.5 | 30 | Sidekiq::Job
|
53
|
+
Sidekiq 7.0.3 | 46.5 sec| 10,850 | 2.7.5 | 30 | ActiveJob 7.0.4
|
54
|
+
|
55
|
+
Most of Sidekiq's overhead is Redis network I/O.
|
56
|
+
ActiveJob adds a notable amount of CPU overhead due to argument deserialization and callbacks.
|
57
|
+
Concurrency of 30 was determined experimentally to maximize one CPU without saturating it.
|
59
58
|
|
60
59
|
Want to Upgrade?
|
61
60
|
-------------------
|
62
61
|
|
62
|
+
Use `bundle up sidekiq` to upgrade Sidekiq and all its dependencies.
|
63
|
+
Upgrade notes between each major version can be found in the `docs/` directory.
|
64
|
+
|
63
65
|
I also sell Sidekiq Pro and Sidekiq Enterprise, extensions to Sidekiq which provide more
|
64
66
|
features, a commercial-friendly license and allow you to support high
|
65
67
|
quality open source development all at the same time. Please see the
|
66
|
-
[Sidekiq](
|
67
|
-
|
68
|
-
Subscribe to the **[quarterly newsletter](https://tinyletter.com/sidekiq)** to stay informed about the latest
|
69
|
-
features and changes to Sidekiq and its bigger siblings.
|
68
|
+
[Sidekiq](https://sidekiq.org/) homepage for more detail.
|
70
69
|
|
71
70
|
|
72
71
|
Problems?
|
73
72
|
-----------------
|
74
73
|
|
75
|
-
**Please do not directly email any Sidekiq committers with questions or problems.**
|
74
|
+
**Please do not directly email any Sidekiq committers with questions or problems.**
|
75
|
+
A community is best served when discussions are held in public.
|
76
76
|
|
77
|
-
If you have a problem, please review the [FAQ](https://github.com/
|
78
|
-
Searching the [issues](https://github.com/
|
77
|
+
If you have a problem, please review the [FAQ](https://github.com/sidekiq/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/sidekiq/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages.
|
78
|
+
Searching the [issues](https://github.com/sidekiq/sidekiq/issues) for your problem is also a good idea.
|
79
79
|
|
80
|
-
Sidekiq Pro and Sidekiq Enterprise customers get private email support.
|
80
|
+
Sidekiq Pro and Sidekiq Enterprise customers get private email support.
|
81
|
+
You can purchase at https://sidekiq.org; email support@contribsys.com for help.
|
81
82
|
|
82
83
|
Useful resources:
|
83
84
|
|
84
|
-
* Product documentation is in the [wiki](https://github.com/
|
85
|
-
*
|
85
|
+
* Product documentation is in the [wiki](https://github.com/sidekiq/sidekiq/wiki).
|
86
|
+
* Occasional announcements are made to the [@sidekiq](https://twitter.com/sidekiq) Twitter account.
|
86
87
|
* The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q & A.
|
87
88
|
|
88
|
-
|
89
|
-
|
90
|
-
Every Friday morning is Sidekiq happy hour: I video chat and answer questions.
|
91
|
-
See the [Sidekiq support page](http://sidekiq.org/support.html) for details.
|
89
|
+
Every Friday morning is Sidekiq office hour: I video chat and answer questions.
|
90
|
+
See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
|
92
91
|
|
93
|
-
|
92
|
+
Contributing
|
94
93
|
-----------------
|
95
94
|
|
96
|
-
|
97
|
-
performance problems on JRuby. Unfortunately MRI does not have good multithreaded profiling tools.
|
98
|
-
|
95
|
+
Please see [the contributing guidelines](https://github.com/sidekiq/sidekiq/blob/main/.github/contributing.md).
|
99
96
|
|
100
97
|
License
|
101
98
|
-----------------
|
102
99
|
|
103
|
-
Please see [LICENSE](https://github.com/
|
104
|
-
|
100
|
+
Please see [LICENSE.txt](https://github.com/sidekiq/sidekiq/blob/main/LICENSE.txt) for licensing details.
|
101
|
+
The license for Sidekiq Pro and Sidekiq Enterprise can be found in [COMM-LICENSE.txt](https://github.com/sidekiq/sidekiq/blob/main/COMM-LICENSE.txt).
|
105
102
|
|
106
103
|
Author
|
107
104
|
-----------------
|
108
105
|
|
109
|
-
Mike Perham, [@
|
106
|
+
Mike Perham, [@getajobmike](https://twitter.com/getajobmike) / [@sidekiq](https://twitter.com/sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
|
data/bin/sidekiq
CHANGED
@@ -4,15 +4,34 @@
|
|
4
4
|
# RUBYOPT=-w bundle exec sidekiq
|
5
5
|
$TESTING = false
|
6
6
|
|
7
|
-
require_relative
|
7
|
+
require_relative "../lib/sidekiq/cli"
|
8
|
+
|
9
|
+
def integrate_with_systemd
|
10
|
+
return unless ENV["NOTIFY_SOCKET"]
|
11
|
+
|
12
|
+
Sidekiq.configure_server do |config|
|
13
|
+
config.logger.info "Enabling systemd notification integration"
|
14
|
+
require "sidekiq/sd_notify"
|
15
|
+
config.on(:startup) do
|
16
|
+
Sidekiq::SdNotify.ready
|
17
|
+
end
|
18
|
+
config.on(:shutdown) do
|
19
|
+
Sidekiq::SdNotify.stopping
|
20
|
+
end
|
21
|
+
Sidekiq.start_watchdog if Sidekiq::SdNotify.watchdog?
|
22
|
+
end
|
23
|
+
end
|
8
24
|
|
9
25
|
begin
|
10
26
|
cli = Sidekiq::CLI.instance
|
11
27
|
cli.parse
|
28
|
+
|
29
|
+
integrate_with_systemd
|
30
|
+
|
12
31
|
cli.run
|
13
32
|
rescue => e
|
14
33
|
raise e if $DEBUG
|
15
|
-
|
16
|
-
|
34
|
+
warn e.message
|
35
|
+
warn e.backtrace.join("\n")
|
17
36
|
exit 1
|
18
37
|
end
|
data/bin/sidekiqload
CHANGED
@@ -1,149 +1,247 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
+
#
|
4
|
+
# bin/sidekiqload is a helpful script to load test and
|
5
|
+
# performance tune Sidekiq's core. It creates 500,000 no-op
|
6
|
+
# jobs and executes them as fast as possible.
|
7
|
+
# Example Usage:
|
8
|
+
#
|
9
|
+
# > RUBY_YJIT_ENABLE=1 LATENCY=0 THREADS=10 bin/sidekiqload
|
10
|
+
# Result: Done, 500000 jobs in 20.264945 sec, 24673 jobs/sec
|
11
|
+
#
|
12
|
+
# Use LATENCY=1 to get a more real world network setup
|
13
|
+
# but you'll need to setup and start toxiproxy as noted below.
|
14
|
+
#
|
15
|
+
# Use AJ=1 to test ActiveJob instead of plain old Sidekiq::Jobs so
|
16
|
+
# you can see the runtime performance difference between the two APIs.
|
17
|
+
#
|
18
|
+
# None of this script is considered a public API and may change over time.
|
19
|
+
#
|
20
|
+
|
3
21
|
# Quiet some warnings we see when running in warning mode:
|
4
22
|
# RUBYOPT=-w bundle exec sidekiq
|
5
23
|
$TESTING = false
|
24
|
+
puts RUBY_DESCRIPTION
|
25
|
+
puts(%w[THREADS LATENCY AJ PROFILE].map { |x| "#{x}: #{ENV[x] || "nil"}" }.join(", "))
|
26
|
+
|
27
|
+
require "ruby-prof" if ENV["PROFILE"]
|
28
|
+
require "bundler/setup"
|
29
|
+
Bundler.require(:default, :load_test)
|
30
|
+
|
31
|
+
latency = Integer(ENV["LATENCY"] || 1)
|
32
|
+
if latency > 0
|
33
|
+
# brew tap shopify/shopify
|
34
|
+
# brew install toxiproxy
|
35
|
+
# run `toxiproxy-server` in a separate terminal window.
|
36
|
+
require "toxiproxy"
|
37
|
+
# simulate a non-localhost network for realer-world conditions.
|
38
|
+
# adding 1ms of network latency has an ENORMOUS impact on benchmarks
|
39
|
+
Toxiproxy.populate([{
|
40
|
+
name: "redis",
|
41
|
+
listen: "127.0.0.1:6380",
|
42
|
+
upstream: "127.0.0.1:6379"
|
43
|
+
}])
|
44
|
+
end
|
6
45
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
include Sidekiq::Util
|
46
|
+
if ENV["AJ"]
|
47
|
+
require "active_job"
|
48
|
+
puts "Using ActiveJob #{ActiveJob::VERSION::STRING}"
|
49
|
+
ActiveJob::Base.queue_adapter = :sidekiq
|
50
|
+
ActiveJob::Base.logger.level = Logger::WARN
|
14
51
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
config.average_scheduled_poll_interval = 2
|
21
|
-
config.reliable! if defined?(Sidekiq::Pro)
|
52
|
+
class LoadJob < ActiveJob::Base
|
53
|
+
def perform(idx, ts = nil)
|
54
|
+
puts(Time.now.to_f - ts) if !ts.nil?
|
55
|
+
end
|
56
|
+
end
|
22
57
|
end
|
23
58
|
|
24
59
|
class LoadWorker
|
25
|
-
include Sidekiq::
|
60
|
+
include Sidekiq::Job
|
26
61
|
sidekiq_options retry: 1
|
27
62
|
sidekiq_retry_in do |x|
|
28
63
|
1
|
29
64
|
end
|
30
65
|
|
31
|
-
def perform(idx)
|
32
|
-
|
66
|
+
def perform(idx, ts = nil)
|
67
|
+
puts(Time.now.to_f - ts) if !ts.nil?
|
68
|
+
# raise idx.to_s if idx % 100 == 1
|
33
69
|
end
|
34
70
|
end
|
35
71
|
|
36
|
-
|
37
|
-
|
38
|
-
# gem install toxiproxy
|
39
|
-
#require 'toxiproxy'
|
40
|
-
# simulate a non-localhost network for realer-world conditions.
|
41
|
-
# adding 1ms of network latency has an ENORMOUS impact on benchmarks
|
42
|
-
#Toxiproxy.populate([{
|
43
|
-
#"name": "redis",
|
44
|
-
#"listen": "127.0.0.1:6380",
|
45
|
-
#"upstream": "127.0.0.1:6379"
|
46
|
-
#}])
|
47
|
-
|
48
|
-
self_read, self_write = IO.pipe
|
49
|
-
%w(INT TERM TSTP TTIN).each do |sig|
|
50
|
-
begin
|
51
|
-
trap sig do
|
52
|
-
self_write.puts(sig)
|
53
|
-
end
|
54
|
-
rescue ArgumentError
|
55
|
-
puts "Signal #{sig} not supported"
|
56
|
-
end
|
72
|
+
def Process.rss
|
73
|
+
`ps -o rss= -p #{Process.pid}`.chomp.to_i
|
57
74
|
end
|
58
75
|
|
59
|
-
|
60
|
-
def
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
# Handle Ctrl-C in JRuby like MRI
|
65
|
-
# http://jira.codehaus.org/browse/JRUBY-4637
|
66
|
-
raise Interrupt
|
67
|
-
when 'TERM'
|
68
|
-
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
69
|
-
raise Interrupt
|
70
|
-
when 'TSTP'
|
71
|
-
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
72
|
-
launcher.quiet
|
73
|
-
when 'TTIN'
|
74
|
-
Thread.list.each do |thread|
|
75
|
-
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['label']}"
|
76
|
-
if thread.backtrace
|
77
|
-
Sidekiq.logger.warn thread.backtrace.join("\n")
|
78
|
-
else
|
79
|
-
Sidekiq.logger.warn "<no backtrace available>"
|
80
|
-
end
|
81
|
-
end
|
76
|
+
class Loader
|
77
|
+
def initialize
|
78
|
+
@iter = ENV["GC"] ? 10 : 500
|
79
|
+
@count = Integer(ENV["COUNT"] || 1_000)
|
80
|
+
@latency = Integer(ENV["LATENCY"] || 1)
|
82
81
|
end
|
83
|
-
end
|
84
82
|
|
85
|
-
def
|
86
|
-
|
87
|
-
|
83
|
+
def configure
|
84
|
+
@x = Sidekiq.configure_embed do |config|
|
85
|
+
config.redis = {db: 13, port: ((@latency > 0) ? 6380 : 6379)}
|
86
|
+
config.concurrency = Integer(ENV.fetch("THREADS", "10"))
|
87
|
+
# config.redis = { db: 13, port: 6380, driver: :hiredis}
|
88
|
+
config.queues = %w[default]
|
89
|
+
config.logger.level = Logger::WARN
|
90
|
+
config.average_scheduled_poll_interval = 2
|
91
|
+
config.reliable! if defined?(Sidekiq::Pro)
|
92
|
+
end
|
88
93
|
|
89
|
-
|
90
|
-
|
94
|
+
@self_read, @self_write = IO.pipe
|
95
|
+
%w[INT TERM TSTP TTIN].each do |sig|
|
96
|
+
trap sig do
|
97
|
+
@self_write.puts(sig)
|
98
|
+
end
|
99
|
+
rescue ArgumentError
|
100
|
+
puts "Signal #{sig} not supported"
|
101
|
+
end
|
102
|
+
end
|
91
103
|
|
92
|
-
|
93
|
-
|
94
|
-
|
104
|
+
def handle_signal(sig)
|
105
|
+
launcher = @x
|
106
|
+
Sidekiq.logger.debug "Got #{sig} signal"
|
107
|
+
case sig
|
108
|
+
when "INT"
|
109
|
+
# Handle Ctrl-C in JRuby like MRI
|
110
|
+
# http://jira.codehaus.org/browse/JRUBY-4637
|
111
|
+
raise Interrupt
|
112
|
+
when "TERM"
|
113
|
+
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
114
|
+
raise Interrupt
|
115
|
+
when "TSTP"
|
116
|
+
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
117
|
+
launcher.quiet
|
118
|
+
when "TTIN"
|
119
|
+
Thread.list.each do |thread|
|
120
|
+
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["label"]}"
|
121
|
+
if thread.backtrace
|
122
|
+
Sidekiq.logger.warn thread.backtrace.join("\n")
|
123
|
+
else
|
124
|
+
Sidekiq.logger.warn "<no backtrace available>"
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
95
128
|
end
|
96
|
-
|
97
|
-
|
129
|
+
|
130
|
+
def setup
|
131
|
+
Sidekiq.logger.error("Setup RSS: #{Process.rss}")
|
132
|
+
Sidekiq.redis { |c| c.flushdb }
|
133
|
+
start = Time.now
|
134
|
+
if ENV["AJ"]
|
135
|
+
@iter.times do
|
136
|
+
@count.times do |idx|
|
137
|
+
LoadJob.perform_later(idx)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
else
|
141
|
+
@iter.times do
|
142
|
+
arr = Array.new(@count) { |idx| [idx] }
|
143
|
+
Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
|
144
|
+
end
|
145
|
+
end
|
146
|
+
Sidekiq.logger.warn "Created #{@count * @iter} jobs in #{Time.now - start} sec"
|
98
147
|
end
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
sleep 1
|
107
|
-
qsize, retries = Sidekiq.redis do |conn|
|
108
|
-
conn.pipelined do
|
148
|
+
|
149
|
+
def monitor
|
150
|
+
@monitor = Thread.new do
|
151
|
+
GC.start
|
152
|
+
loop do
|
153
|
+
sleep 0.2
|
154
|
+
qsize = Sidekiq.redis do |conn|
|
109
155
|
conn.llen "queue:default"
|
110
|
-
conn.zcard "retry"
|
111
156
|
end
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
157
|
+
total = qsize
|
158
|
+
if total == 0
|
159
|
+
ending = Time.now - @start
|
160
|
+
size = @iter * @count
|
161
|
+
Sidekiq.logger.error("Done, #{size} jobs in #{ending} sec, #{(size / ending).to_i} jobs/sec")
|
162
|
+
Sidekiq.logger.error("Ending RSS: #{Process.rss}")
|
163
|
+
Sidekiq.logger.error("Now here's the latency for three jobs")
|
164
|
+
|
165
|
+
if ENV["AJ"]
|
166
|
+
LoadJob.perform_later(1, Time.now.to_f)
|
167
|
+
LoadJob.perform_later(2, Time.now.to_f)
|
168
|
+
LoadJob.perform_later(3, Time.now.to_f)
|
169
|
+
else
|
170
|
+
LoadWorker.perform_async(1, Time.now.to_f)
|
171
|
+
LoadWorker.perform_async(2, Time.now.to_f)
|
172
|
+
LoadWorker.perform_async(3, Time.now.to_f)
|
173
|
+
end
|
174
|
+
|
175
|
+
sleep 0.1
|
176
|
+
@x.stop
|
177
|
+
Process.kill("INT", $$)
|
178
|
+
break
|
179
|
+
end
|
119
180
|
end
|
120
181
|
end
|
121
182
|
end
|
122
|
-
end
|
123
183
|
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
184
|
+
def with_latency(latency, &block)
|
185
|
+
Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
|
186
|
+
if latency > 0
|
187
|
+
Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
|
188
|
+
else
|
189
|
+
yield
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
def run(name)
|
194
|
+
Sidekiq.logger.warn("Starting #{name}")
|
195
|
+
monitor
|
196
|
+
|
197
|
+
if ENV["PROFILE"]
|
198
|
+
RubyProf.exclude_threads = [@monitor]
|
199
|
+
RubyProf.start
|
200
|
+
elsif ENV["GC"]
|
201
|
+
GC.start
|
202
|
+
GC.compact
|
203
|
+
GC.disable
|
204
|
+
Sidekiq.logger.error("GC Start RSS: #{Process.rss}")
|
136
205
|
end
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
206
|
+
@start = Time.now
|
207
|
+
with_latency(@latency) do
|
208
|
+
@x.run
|
209
|
+
|
210
|
+
while (readable_io = IO.select([@self_read]))
|
211
|
+
signal = readable_io.first[0].gets.strip
|
212
|
+
handle_signal(signal)
|
213
|
+
end
|
214
|
+
end
|
215
|
+
# normal
|
216
|
+
rescue Interrupt
|
217
|
+
rescue => e
|
218
|
+
raise e if $DEBUG
|
219
|
+
warn e.message
|
220
|
+
warn e.backtrace.join("\n")
|
221
|
+
exit 1
|
222
|
+
ensure
|
223
|
+
@x.stop
|
224
|
+
end
|
225
|
+
|
226
|
+
def done
|
227
|
+
Sidekiq.logger.error("GC End RSS: #{Process.rss}") if ENV["GC"]
|
228
|
+
if ENV["PROFILE"]
|
229
|
+
Sidekiq.logger.error("Profiling...")
|
230
|
+
result = RubyProf.stop
|
231
|
+
printer = RubyProf::GraphHtmlPrinter.new(result)
|
232
|
+
printer.print(File.new("output.html", "w"), min_percent: 1)
|
233
|
+
end
|
234
|
+
end
|
149
235
|
end
|
236
|
+
|
237
|
+
ll = Loader.new
|
238
|
+
ll.configure
|
239
|
+
|
240
|
+
if ENV["WARM"]
|
241
|
+
ll.setup
|
242
|
+
ll.run("warmup")
|
243
|
+
end
|
244
|
+
|
245
|
+
ll.setup
|
246
|
+
ll.run("load")
|
247
|
+
ll.done
|
data/bin/sidekiqmon
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "sidekiq/monitor"
|
4
|
+
|
5
|
+
# disable the Redis connection pool logging
|
6
|
+
Sidekiq.default_configuration.logger.level = :warn
|
7
|
+
|
8
|
+
section = "all"
|
9
|
+
section = ARGV[0] if ARGV.size == 1
|
10
|
+
|
11
|
+
Sidekiq::Monitor::Status.new.display(section)
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require "rails/generators/named_base"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
module Generators # :nodoc:
|
5
|
+
class JobGenerator < ::Rails::Generators::NamedBase # :nodoc:
|
6
|
+
desc "This generator creates a Sidekiq Job in app/sidekiq and a corresponding test"
|
7
|
+
|
8
|
+
check_class_collision suffix: "Job"
|
9
|
+
|
10
|
+
def self.default_generator_root
|
11
|
+
File.dirname(__FILE__)
|
12
|
+
end
|
13
|
+
|
14
|
+
def create_job_file
|
15
|
+
template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
|
16
|
+
end
|
17
|
+
|
18
|
+
def create_test_file
|
19
|
+
return unless test_framework
|
20
|
+
|
21
|
+
if test_framework == :rspec
|
22
|
+
create_job_spec
|
23
|
+
else
|
24
|
+
create_job_test
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def create_job_spec
|
31
|
+
template_file = File.join(
|
32
|
+
"spec/sidekiq",
|
33
|
+
class_path,
|
34
|
+
"#{file_name}_job_spec.rb"
|
35
|
+
)
|
36
|
+
template "job_spec.rb.erb", template_file
|
37
|
+
end
|
38
|
+
|
39
|
+
def create_job_test
|
40
|
+
template_file = File.join(
|
41
|
+
"test/sidekiq",
|
42
|
+
class_path,
|
43
|
+
"#{file_name}_job_test.rb"
|
44
|
+
)
|
45
|
+
template "job_test.rb.erb", template_file
|
46
|
+
end
|
47
|
+
|
48
|
+
def file_name
|
49
|
+
@_file_name ||= super.sub(/_?job\z/i, "")
|
50
|
+
end
|
51
|
+
|
52
|
+
def test_framework
|
53
|
+
::Rails.application.config.generators.options[:rails][:test_framework]
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'test_helper'
|
2
2
|
<% module_namespacing do -%>
|
3
|
-
class <%= class_name %>
|
3
|
+
class <%= class_name %>JobTest < Minitest::Test
|
4
4
|
def test_example
|
5
5
|
skip "add some examples to (or delete) #{__FILE__}"
|
6
6
|
end
|