sidekiq 5.2.9 → 7.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Changes.md +547 -8
- data/LICENSE.txt +9 -0
- data/README.md +47 -50
- data/bin/sidekiq +22 -3
- data/bin/sidekiqload +213 -115
- data/bin/sidekiqmon +11 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +558 -353
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +204 -226
- data/lib/sidekiq/client.rb +124 -101
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +278 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +49 -42
- data/lib/sidekiq/job.rb +374 -0
- data/lib/sidekiq/job_logger.rb +33 -7
- data/lib/sidekiq/job_retry.rb +145 -107
- data/lib/sidekiq/job_util.rb +107 -0
- data/lib/sidekiq/launcher.rb +203 -105
- data/lib/sidekiq/logger.rb +131 -0
- data/lib/sidekiq/manager.rb +43 -46
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +113 -56
- data/lib/sidekiq/middleware/current_attributes.rb +95 -0
- data/lib/sidekiq/middleware/i18n.rb +7 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +146 -0
- data/lib/sidekiq/paginator.rb +28 -16
- data/lib/sidekiq/processor.rb +122 -120
- data/lib/sidekiq/rails.rb +53 -38
- data/lib/sidekiq/redis_client_adapter.rb +96 -0
- data/lib/sidekiq/redis_connection.rb +38 -107
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +111 -49
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +6 -5
- data/lib/sidekiq/testing.rb +66 -84
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +3 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +123 -79
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +137 -106
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +68 -107
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +92 -182
- data/sidekiq.gemspec +25 -16
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +131 -61
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +168 -0
- data/web/assets/javascripts/dashboard.js +36 -292
- data/web/assets/javascripts/metrics.js +264 -0
- data/web/assets/stylesheets/application-dark.css +147 -0
- data/web/assets/stylesheets/application-rtl.css +2 -95
- data/web/assets/stylesheets/application.css +102 -522
- data/web/locales/ar.yml +71 -65
- data/web/locales/cs.yml +62 -62
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -53
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +84 -66
- data/web/locales/es.yml +70 -54
- data/web/locales/fa.yml +65 -65
- data/web/locales/fr.yml +83 -62
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +65 -64
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +75 -64
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +83 -0
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +83 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +68 -63
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +64 -64
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +43 -16
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +6 -3
- data/web/views/_job_info.erb +21 -4
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +1 -1
- data/web/views/_paging.erb +2 -0
- data/web/views/_poll_link.erb +3 -6
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +75 -25
- data/web/views/dashboard.erb +58 -18
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +3 -1
- data/web/views/metrics.erb +82 -0
- data/web/views/metrics_for_job.erb +68 -0
- data/web/views/morgue.erb +14 -15
- data/web/views/queue.erb +33 -24
- data/web/views/queues.erb +13 -3
- data/web/views/retries.erb +16 -17
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +17 -15
- metadata +71 -72
- data/.circleci/config.yml +0 -61
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -15
- data/.travis.yml +0 -11
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/COMM-LICENSE +0 -97
- data/Ent-Changes.md +0 -238
- data/Gemfile +0 -23
- data/LICENSE +0 -9
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-Changes.md +0 -759
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -20
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/ctl.rb +0 -221
- data/lib/sidekiq/delay.rb +0 -42
- data/lib/sidekiq/exception_handler.rb +0 -29
- data/lib/sidekiq/extensions/action_mailer.rb +0 -57
- data/lib/sidekiq/extensions/active_record.rb +0 -40
- data/lib/sidekiq/extensions/class_methods.rb +0 -40
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -31
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
- data/lib/sidekiq/util.rb +0 -66
- data/lib/sidekiq/worker.rb +0 -220
data/README.md
CHANGED
@@ -2,10 +2,7 @@ Sidekiq
|
|
2
2
|
==============
|
3
3
|
|
4
4
|
[![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
|
5
|
-
|
6
|
-
[![Build Status](https://circleci.com/gh/mperham/sidekiq/tree/master.svg?style=svg)](https://circleci.com/gh/mperham/sidekiq/tree/master)
|
7
|
-
[![Gitter Chat](https://badges.gitter.im/mperham/sidekiq.svg)](https://gitter.im/mperham/sidekiq)
|
8
|
-
|
5
|
+
![Build](https://github.com/sidekiq/sidekiq/workflows/CI/badge.svg)
|
9
6
|
|
10
7
|
Simple, efficient background processing for Ruby.
|
11
8
|
|
@@ -13,97 +10,97 @@ Sidekiq uses threads to handle many jobs at the same time in the
|
|
13
10
|
same process. It does not require Rails but will integrate tightly with
|
14
11
|
Rails to make background processing dead simple.
|
15
12
|
|
16
|
-
Sidekiq is compatible with Resque. It uses the exact same
|
17
|
-
message format as Resque so it can integrate into an existing Resque processing farm.
|
18
|
-
You can have Sidekiq and Resque run side-by-side at the same time and
|
19
|
-
use the Resque client to enqueue jobs in Redis to be processed by Sidekiq.
|
20
|
-
|
21
|
-
Performance
|
22
|
-
---------------
|
23
|
-
|
24
|
-
Version | Latency | Garbage created for 10,000 jobs | Time to process 100,000 jobs | Throughput
|
25
|
-
-----------------|------|---------|---------|------------------------
|
26
|
-
Sidekiq 4.0.0 | 10ms | 151 MB | 22 sec | **4500 jobs/sec**
|
27
|
-
Sidekiq 3.5.1 | 22ms | 1257 MB | 125 sec | 800 jobs/sec
|
28
|
-
Resque 1.25.2 | - | - | 420 sec | 240 jobs/sec
|
29
|
-
DelayedJob 4.1.1 | - | - | 465 sec | 215 jobs/sec
|
30
|
-
|
31
|
-
<small>This benchmark can be found in `bin/sidekiqload`.</small>
|
32
13
|
|
33
14
|
Requirements
|
34
15
|
-----------------
|
35
16
|
|
36
|
-
|
37
|
-
|
38
|
-
All Rails releases >= 4.0 are officially supported.
|
17
|
+
- Redis: 6.2+
|
18
|
+
- Ruby: MRI 2.7+ or JRuby 9.3+.
|
39
19
|
|
40
|
-
|
41
|
-
installations with thousands of worker threads.
|
20
|
+
Sidekiq 7.0 supports Rails 6.0+ but does not require it.
|
42
21
|
|
43
22
|
|
44
23
|
Installation
|
45
24
|
-----------------
|
46
25
|
|
47
|
-
|
26
|
+
bundle add sidekiq
|
48
27
|
|
49
28
|
|
50
29
|
Getting Started
|
51
30
|
-----------------
|
52
31
|
|
53
|
-
See the [Getting Started wiki page](https://github.com/
|
54
|
-
You can watch [this
|
32
|
+
See the [Getting Started wiki page](https://github.com/sidekiq/sidekiq/wiki/Getting-Started) and follow the simple setup process.
|
33
|
+
You can watch [this YouTube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
|
55
34
|
Sidekiq and see its features in action. Here's the Web UI:
|
56
35
|
|
57
|
-
![Web UI](https://github.com/
|
36
|
+
![Web UI](https://github.com/sidekiq/sidekiq/raw/main/examples/web-ui.png)
|
58
37
|
|
38
|
+
Performance
|
39
|
+
---------------
|
40
|
+
|
41
|
+
The benchmark in `bin/sidekiqload` creates 500,000 no-op jobs and drains them as fast as possible, assuming a fixed Redis network latency of 1ms.
|
42
|
+
This requires a lot of Redis network I/O and JSON parsing.
|
43
|
+
This benchmark is IO-bound so we increase the concurrency to 25.
|
44
|
+
If your application is sending lots of emails or performing other network-intensive work, you could see a similar benefit but be careful not to saturate the CPU.
|
45
|
+
|
46
|
+
Version | Time to process 500k jobs | Throughput (jobs/sec) | Ruby | Concurrency | Job Type
|
47
|
+
-----------------|------|---------|---------|------------------------|---
|
48
|
+
Sidekiq 7.0.3 | 21.3 sec| 23,500 | 3.2.0+yjit | 30 | Sidekiq::Job
|
49
|
+
Sidekiq 7.0.3 | 33.8 sec| 14,700 | 3.2.0+yjit | 30 | ActiveJob 7.0.4
|
50
|
+
Sidekiq 7.0.3 | 23.5 sec| 21,300 | 3.2.0 | 30 | Sidekiq::Job
|
51
|
+
Sidekiq 7.0.3 | 46.5 sec| 10,700 | 3.2.0 | 30 | ActiveJob 7.0.4
|
52
|
+
Sidekiq 7.0.3 | 23.0 sec| 21,700 | 2.7.5 | 30 | Sidekiq::Job
|
53
|
+
Sidekiq 7.0.3 | 46.5 sec| 10,850 | 2.7.5 | 30 | ActiveJob 7.0.4
|
54
|
+
|
55
|
+
Most of Sidekiq's overhead is Redis network I/O.
|
56
|
+
ActiveJob adds a notable amount of CPU overhead due to argument deserialization and callbacks.
|
57
|
+
Concurrency of 30 was determined experimentally to maximize one CPU without saturating it.
|
59
58
|
|
60
59
|
Want to Upgrade?
|
61
60
|
-------------------
|
62
61
|
|
62
|
+
Use `bundle up sidekiq` to upgrade Sidekiq and all its dependencies.
|
63
|
+
Upgrade notes between each major version can be found in the `docs/` directory.
|
64
|
+
|
63
65
|
I also sell Sidekiq Pro and Sidekiq Enterprise, extensions to Sidekiq which provide more
|
64
66
|
features, a commercial-friendly license and allow you to support high
|
65
67
|
quality open source development all at the same time. Please see the
|
66
|
-
[Sidekiq](
|
67
|
-
|
68
|
-
Subscribe to the **[quarterly newsletter](https://tinyletter.com/sidekiq)** to stay informed about the latest
|
69
|
-
features and changes to Sidekiq and its bigger siblings.
|
68
|
+
[Sidekiq](https://sidekiq.org/) homepage for more detail.
|
70
69
|
|
71
70
|
|
72
71
|
Problems?
|
73
72
|
-----------------
|
74
73
|
|
75
|
-
**Please do not directly email any Sidekiq committers with questions or problems.**
|
74
|
+
**Please do not directly email any Sidekiq committers with questions or problems.**
|
75
|
+
A community is best served when discussions are held in public.
|
76
76
|
|
77
|
-
If you have a problem, please review the [FAQ](https://github.com/
|
78
|
-
Searching the [issues](https://github.com/
|
77
|
+
If you have a problem, please review the [FAQ](https://github.com/sidekiq/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/sidekiq/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages.
|
78
|
+
Searching the [issues](https://github.com/sidekiq/sidekiq/issues) for your problem is also a good idea.
|
79
79
|
|
80
|
-
Sidekiq Pro and Sidekiq Enterprise customers get private email support.
|
80
|
+
Sidekiq Pro and Sidekiq Enterprise customers get private email support.
|
81
|
+
You can purchase at https://sidekiq.org; email support@contribsys.com for help.
|
81
82
|
|
82
83
|
Useful resources:
|
83
84
|
|
84
|
-
* Product documentation is in the [wiki](https://github.com/
|
85
|
-
*
|
85
|
+
* Product documentation is in the [wiki](https://github.com/sidekiq/sidekiq/wiki).
|
86
|
+
* Occasional announcements are made to the [@sidekiq](https://twitter.com/sidekiq) Twitter account.
|
86
87
|
* The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q & A.
|
87
88
|
|
88
|
-
|
89
|
-
|
90
|
-
Every Friday morning is Sidekiq happy hour: I video chat and answer questions.
|
91
|
-
See the [Sidekiq support page](http://sidekiq.org/support.html) for details.
|
89
|
+
Every Friday morning is Sidekiq office hour: I video chat and answer questions.
|
90
|
+
See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
|
92
91
|
|
93
|
-
|
92
|
+
Contributing
|
94
93
|
-----------------
|
95
94
|
|
96
|
-
|
97
|
-
performance problems on JRuby. Unfortunately MRI does not have good multithreaded profiling tools.
|
98
|
-
|
95
|
+
Please see [the contributing guidelines](https://github.com/sidekiq/sidekiq/blob/main/.github/contributing.md).
|
99
96
|
|
100
97
|
License
|
101
98
|
-----------------
|
102
99
|
|
103
|
-
Please see [LICENSE](https://github.com/
|
104
|
-
|
100
|
+
Please see [LICENSE.txt](https://github.com/sidekiq/sidekiq/blob/main/LICENSE.txt) for licensing details.
|
101
|
+
The license for Sidekiq Pro and Sidekiq Enterprise can be found in [COMM-LICENSE.txt](https://github.com/sidekiq/sidekiq/blob/main/COMM-LICENSE.txt).
|
105
102
|
|
106
103
|
Author
|
107
104
|
-----------------
|
108
105
|
|
109
|
-
Mike Perham, [@
|
106
|
+
Mike Perham, [@getajobmike](https://twitter.com/getajobmike) / [@sidekiq](https://twitter.com/sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
|
data/bin/sidekiq
CHANGED
@@ -4,15 +4,34 @@
|
|
4
4
|
# RUBYOPT=-w bundle exec sidekiq
|
5
5
|
$TESTING = false
|
6
6
|
|
7
|
-
require_relative
|
7
|
+
require_relative "../lib/sidekiq/cli"
|
8
|
+
|
9
|
+
def integrate_with_systemd
|
10
|
+
return unless ENV["NOTIFY_SOCKET"]
|
11
|
+
|
12
|
+
Sidekiq.configure_server do |config|
|
13
|
+
config.logger.info "Enabling systemd notification integration"
|
14
|
+
require "sidekiq/sd_notify"
|
15
|
+
config.on(:startup) do
|
16
|
+
Sidekiq::SdNotify.ready
|
17
|
+
end
|
18
|
+
config.on(:shutdown) do
|
19
|
+
Sidekiq::SdNotify.stopping
|
20
|
+
end
|
21
|
+
Sidekiq.start_watchdog if Sidekiq::SdNotify.watchdog?
|
22
|
+
end
|
23
|
+
end
|
8
24
|
|
9
25
|
begin
|
10
26
|
cli = Sidekiq::CLI.instance
|
11
27
|
cli.parse
|
28
|
+
|
29
|
+
integrate_with_systemd
|
30
|
+
|
12
31
|
cli.run
|
13
32
|
rescue => e
|
14
33
|
raise e if $DEBUG
|
15
|
-
|
16
|
-
|
34
|
+
warn e.message
|
35
|
+
warn e.backtrace.join("\n")
|
17
36
|
exit 1
|
18
37
|
end
|
data/bin/sidekiqload
CHANGED
@@ -1,149 +1,247 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
+
#
|
4
|
+
# bin/sidekiqload is a helpful script to load test and
|
5
|
+
# performance tune Sidekiq's core. It creates 500,000 no-op
|
6
|
+
# jobs and executes them as fast as possible.
|
7
|
+
# Example Usage:
|
8
|
+
#
|
9
|
+
# > RUBY_YJIT_ENABLE=1 LATENCY=0 THREADS=10 bin/sidekiqload
|
10
|
+
# Result: Done, 500000 jobs in 20.264945 sec, 24673 jobs/sec
|
11
|
+
#
|
12
|
+
# Use LATENCY=1 to get a more real world network setup
|
13
|
+
# but you'll need to setup and start toxiproxy as noted below.
|
14
|
+
#
|
15
|
+
# Use AJ=1 to test ActiveJob instead of plain old Sidekiq::Jobs so
|
16
|
+
# you can see the runtime performance difference between the two APIs.
|
17
|
+
#
|
18
|
+
# None of this script is considered a public API and may change over time.
|
19
|
+
#
|
20
|
+
|
3
21
|
# Quiet some warnings we see when running in warning mode:
|
4
22
|
# RUBYOPT=-w bundle exec sidekiq
|
5
23
|
$TESTING = false
|
24
|
+
puts RUBY_DESCRIPTION
|
25
|
+
puts(%w[THREADS LATENCY AJ PROFILE].map { |x| "#{x}: #{ENV[x] || "nil"}" }.join(", "))
|
26
|
+
|
27
|
+
require "ruby-prof" if ENV["PROFILE"]
|
28
|
+
require "bundler/setup"
|
29
|
+
Bundler.require(:default, :load_test)
|
30
|
+
|
31
|
+
latency = Integer(ENV["LATENCY"] || 1)
|
32
|
+
if latency > 0
|
33
|
+
# brew tap shopify/shopify
|
34
|
+
# brew install toxiproxy
|
35
|
+
# run `toxiproxy-server` in a separate terminal window.
|
36
|
+
require "toxiproxy"
|
37
|
+
# simulate a non-localhost network for realer-world conditions.
|
38
|
+
# adding 1ms of network latency has an ENORMOUS impact on benchmarks
|
39
|
+
Toxiproxy.populate([{
|
40
|
+
name: "redis",
|
41
|
+
listen: "127.0.0.1:6380",
|
42
|
+
upstream: "127.0.0.1:6379"
|
43
|
+
}])
|
44
|
+
end
|
6
45
|
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
include Sidekiq::Util
|
46
|
+
if ENV["AJ"]
|
47
|
+
require "active_job"
|
48
|
+
puts "Using ActiveJob #{ActiveJob::VERSION::STRING}"
|
49
|
+
ActiveJob::Base.queue_adapter = :sidekiq
|
50
|
+
ActiveJob::Base.logger.level = Logger::WARN
|
14
51
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
config.average_scheduled_poll_interval = 2
|
21
|
-
config.reliable! if defined?(Sidekiq::Pro)
|
52
|
+
class LoadJob < ActiveJob::Base
|
53
|
+
def perform(idx, ts = nil)
|
54
|
+
puts(Time.now.to_f - ts) if !ts.nil?
|
55
|
+
end
|
56
|
+
end
|
22
57
|
end
|
23
58
|
|
24
59
|
class LoadWorker
|
25
|
-
include Sidekiq::
|
60
|
+
include Sidekiq::Job
|
26
61
|
sidekiq_options retry: 1
|
27
62
|
sidekiq_retry_in do |x|
|
28
63
|
1
|
29
64
|
end
|
30
65
|
|
31
|
-
def perform(idx)
|
32
|
-
|
66
|
+
def perform(idx, ts = nil)
|
67
|
+
puts(Time.now.to_f - ts) if !ts.nil?
|
68
|
+
# raise idx.to_s if idx % 100 == 1
|
33
69
|
end
|
34
70
|
end
|
35
71
|
|
36
|
-
|
37
|
-
|
38
|
-
# gem install toxiproxy
|
39
|
-
#require 'toxiproxy'
|
40
|
-
# simulate a non-localhost network for realer-world conditions.
|
41
|
-
# adding 1ms of network latency has an ENORMOUS impact on benchmarks
|
42
|
-
#Toxiproxy.populate([{
|
43
|
-
#"name": "redis",
|
44
|
-
#"listen": "127.0.0.1:6380",
|
45
|
-
#"upstream": "127.0.0.1:6379"
|
46
|
-
#}])
|
47
|
-
|
48
|
-
self_read, self_write = IO.pipe
|
49
|
-
%w(INT TERM TSTP TTIN).each do |sig|
|
50
|
-
begin
|
51
|
-
trap sig do
|
52
|
-
self_write.puts(sig)
|
53
|
-
end
|
54
|
-
rescue ArgumentError
|
55
|
-
puts "Signal #{sig} not supported"
|
56
|
-
end
|
72
|
+
def Process.rss
|
73
|
+
`ps -o rss= -p #{Process.pid}`.chomp.to_i
|
57
74
|
end
|
58
75
|
|
59
|
-
|
60
|
-
def
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
# Handle Ctrl-C in JRuby like MRI
|
65
|
-
# http://jira.codehaus.org/browse/JRUBY-4637
|
66
|
-
raise Interrupt
|
67
|
-
when 'TERM'
|
68
|
-
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
69
|
-
raise Interrupt
|
70
|
-
when 'TSTP'
|
71
|
-
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
72
|
-
launcher.quiet
|
73
|
-
when 'TTIN'
|
74
|
-
Thread.list.each do |thread|
|
75
|
-
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['label']}"
|
76
|
-
if thread.backtrace
|
77
|
-
Sidekiq.logger.warn thread.backtrace.join("\n")
|
78
|
-
else
|
79
|
-
Sidekiq.logger.warn "<no backtrace available>"
|
80
|
-
end
|
81
|
-
end
|
76
|
+
class Loader
|
77
|
+
def initialize
|
78
|
+
@iter = ENV["GC"] ? 10 : 500
|
79
|
+
@count = Integer(ENV["COUNT"] || 1_000)
|
80
|
+
@latency = Integer(ENV["LATENCY"] || 1)
|
82
81
|
end
|
83
|
-
end
|
84
82
|
|
85
|
-
def
|
86
|
-
|
87
|
-
|
83
|
+
def configure
|
84
|
+
@x = Sidekiq.configure_embed do |config|
|
85
|
+
config.redis = {db: 13, port: ((@latency > 0) ? 6380 : 6379)}
|
86
|
+
config.concurrency = Integer(ENV.fetch("THREADS", "10"))
|
87
|
+
# config.redis = { db: 13, port: 6380, driver: :hiredis}
|
88
|
+
config.queues = %w[default]
|
89
|
+
config.logger.level = Logger::WARN
|
90
|
+
config.average_scheduled_poll_interval = 2
|
91
|
+
config.reliable! if defined?(Sidekiq::Pro)
|
92
|
+
end
|
88
93
|
|
89
|
-
|
90
|
-
|
94
|
+
@self_read, @self_write = IO.pipe
|
95
|
+
%w[INT TERM TSTP TTIN].each do |sig|
|
96
|
+
trap sig do
|
97
|
+
@self_write.puts(sig)
|
98
|
+
end
|
99
|
+
rescue ArgumentError
|
100
|
+
puts "Signal #{sig} not supported"
|
101
|
+
end
|
102
|
+
end
|
91
103
|
|
92
|
-
|
93
|
-
|
94
|
-
|
104
|
+
def handle_signal(sig)
|
105
|
+
launcher = @x
|
106
|
+
Sidekiq.logger.debug "Got #{sig} signal"
|
107
|
+
case sig
|
108
|
+
when "INT"
|
109
|
+
# Handle Ctrl-C in JRuby like MRI
|
110
|
+
# http://jira.codehaus.org/browse/JRUBY-4637
|
111
|
+
raise Interrupt
|
112
|
+
when "TERM"
|
113
|
+
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
114
|
+
raise Interrupt
|
115
|
+
when "TSTP"
|
116
|
+
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
117
|
+
launcher.quiet
|
118
|
+
when "TTIN"
|
119
|
+
Thread.list.each do |thread|
|
120
|
+
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["label"]}"
|
121
|
+
if thread.backtrace
|
122
|
+
Sidekiq.logger.warn thread.backtrace.join("\n")
|
123
|
+
else
|
124
|
+
Sidekiq.logger.warn "<no backtrace available>"
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
95
128
|
end
|
96
|
-
|
97
|
-
|
129
|
+
|
130
|
+
def setup
|
131
|
+
Sidekiq.logger.error("Setup RSS: #{Process.rss}")
|
132
|
+
Sidekiq.redis { |c| c.flushdb }
|
133
|
+
start = Time.now
|
134
|
+
if ENV["AJ"]
|
135
|
+
@iter.times do
|
136
|
+
@count.times do |idx|
|
137
|
+
LoadJob.perform_later(idx)
|
138
|
+
end
|
139
|
+
end
|
140
|
+
else
|
141
|
+
@iter.times do
|
142
|
+
arr = Array.new(@count) { |idx| [idx] }
|
143
|
+
Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
|
144
|
+
end
|
145
|
+
end
|
146
|
+
Sidekiq.logger.warn "Created #{@count * @iter} jobs in #{Time.now - start} sec"
|
98
147
|
end
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
sleep 1
|
107
|
-
qsize, retries = Sidekiq.redis do |conn|
|
108
|
-
conn.pipelined do
|
148
|
+
|
149
|
+
def monitor
|
150
|
+
@monitor = Thread.new do
|
151
|
+
GC.start
|
152
|
+
loop do
|
153
|
+
sleep 0.2
|
154
|
+
qsize = Sidekiq.redis do |conn|
|
109
155
|
conn.llen "queue:default"
|
110
|
-
conn.zcard "retry"
|
111
156
|
end
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
157
|
+
total = qsize
|
158
|
+
if total == 0
|
159
|
+
ending = Time.now - @start
|
160
|
+
size = @iter * @count
|
161
|
+
Sidekiq.logger.error("Done, #{size} jobs in #{ending} sec, #{(size / ending).to_i} jobs/sec")
|
162
|
+
Sidekiq.logger.error("Ending RSS: #{Process.rss}")
|
163
|
+
Sidekiq.logger.error("Now here's the latency for three jobs")
|
164
|
+
|
165
|
+
if ENV["AJ"]
|
166
|
+
LoadJob.perform_later(1, Time.now.to_f)
|
167
|
+
LoadJob.perform_later(2, Time.now.to_f)
|
168
|
+
LoadJob.perform_later(3, Time.now.to_f)
|
169
|
+
else
|
170
|
+
LoadWorker.perform_async(1, Time.now.to_f)
|
171
|
+
LoadWorker.perform_async(2, Time.now.to_f)
|
172
|
+
LoadWorker.perform_async(3, Time.now.to_f)
|
173
|
+
end
|
174
|
+
|
175
|
+
sleep 0.1
|
176
|
+
@x.stop
|
177
|
+
Process.kill("INT", $$)
|
178
|
+
break
|
179
|
+
end
|
119
180
|
end
|
120
181
|
end
|
121
182
|
end
|
122
|
-
end
|
123
183
|
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
184
|
+
def with_latency(latency, &block)
|
185
|
+
Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
|
186
|
+
if latency > 0
|
187
|
+
Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
|
188
|
+
else
|
189
|
+
yield
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
def run(name)
|
194
|
+
Sidekiq.logger.warn("Starting #{name}")
|
195
|
+
monitor
|
196
|
+
|
197
|
+
if ENV["PROFILE"]
|
198
|
+
RubyProf.exclude_threads = [@monitor]
|
199
|
+
RubyProf.start
|
200
|
+
elsif ENV["GC"]
|
201
|
+
GC.start
|
202
|
+
GC.compact
|
203
|
+
GC.disable
|
204
|
+
Sidekiq.logger.error("GC Start RSS: #{Process.rss}")
|
136
205
|
end
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
206
|
+
@start = Time.now
|
207
|
+
with_latency(@latency) do
|
208
|
+
@x.run
|
209
|
+
|
210
|
+
while (readable_io = IO.select([@self_read]))
|
211
|
+
signal = readable_io.first[0].gets.strip
|
212
|
+
handle_signal(signal)
|
213
|
+
end
|
214
|
+
end
|
215
|
+
# normal
|
216
|
+
rescue Interrupt
|
217
|
+
rescue => e
|
218
|
+
raise e if $DEBUG
|
219
|
+
warn e.message
|
220
|
+
warn e.backtrace.join("\n")
|
221
|
+
exit 1
|
222
|
+
ensure
|
223
|
+
@x.stop
|
224
|
+
end
|
225
|
+
|
226
|
+
def done
|
227
|
+
Sidekiq.logger.error("GC End RSS: #{Process.rss}") if ENV["GC"]
|
228
|
+
if ENV["PROFILE"]
|
229
|
+
Sidekiq.logger.error("Profiling...")
|
230
|
+
result = RubyProf.stop
|
231
|
+
printer = RubyProf::GraphHtmlPrinter.new(result)
|
232
|
+
printer.print(File.new("output.html", "w"), min_percent: 1)
|
233
|
+
end
|
234
|
+
end
|
149
235
|
end
|
236
|
+
|
237
|
+
ll = Loader.new
|
238
|
+
ll.configure
|
239
|
+
|
240
|
+
if ENV["WARM"]
|
241
|
+
ll.setup
|
242
|
+
ll.run("warmup")
|
243
|
+
end
|
244
|
+
|
245
|
+
ll.setup
|
246
|
+
ll.run("load")
|
247
|
+
ll.done
|
data/bin/sidekiqmon
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "sidekiq/monitor"
|
4
|
+
|
5
|
+
# disable the Redis connection pool logging
|
6
|
+
Sidekiq.default_configuration.logger.level = :warn
|
7
|
+
|
8
|
+
section = "all"
|
9
|
+
section = ARGV[0] if ARGV.size == 1
|
10
|
+
|
11
|
+
Sidekiq::Monitor::Status.new.display(section)
|
@@ -0,0 +1,57 @@
|
|
1
|
+
require "rails/generators/named_base"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
module Generators # :nodoc:
|
5
|
+
class JobGenerator < ::Rails::Generators::NamedBase # :nodoc:
|
6
|
+
desc "This generator creates a Sidekiq Job in app/sidekiq and a corresponding test"
|
7
|
+
|
8
|
+
check_class_collision suffix: "Job"
|
9
|
+
|
10
|
+
def self.default_generator_root
|
11
|
+
File.dirname(__FILE__)
|
12
|
+
end
|
13
|
+
|
14
|
+
def create_job_file
|
15
|
+
template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
|
16
|
+
end
|
17
|
+
|
18
|
+
def create_test_file
|
19
|
+
return unless test_framework
|
20
|
+
|
21
|
+
if test_framework == :rspec
|
22
|
+
create_job_spec
|
23
|
+
else
|
24
|
+
create_job_test
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def create_job_spec
|
31
|
+
template_file = File.join(
|
32
|
+
"spec/sidekiq",
|
33
|
+
class_path,
|
34
|
+
"#{file_name}_job_spec.rb"
|
35
|
+
)
|
36
|
+
template "job_spec.rb.erb", template_file
|
37
|
+
end
|
38
|
+
|
39
|
+
def create_job_test
|
40
|
+
template_file = File.join(
|
41
|
+
"test/sidekiq",
|
42
|
+
class_path,
|
43
|
+
"#{file_name}_job_test.rb"
|
44
|
+
)
|
45
|
+
template "job_test.rb.erb", template_file
|
46
|
+
end
|
47
|
+
|
48
|
+
def file_name
|
49
|
+
@_file_name ||= super.sub(/_?job\z/i, "")
|
50
|
+
end
|
51
|
+
|
52
|
+
def test_framework
|
53
|
+
::Rails.application.config.generators.options[:rails][:test_framework]
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'test_helper'
|
2
2
|
<% module_namespacing do -%>
|
3
|
-
class <%= class_name %>
|
3
|
+
class <%= class_name %>JobTest < Minitest::Test
|
4
4
|
def test_example
|
5
5
|
skip "add some examples to (or delete) #{__FILE__}"
|
6
6
|
end
|