sidekiq 5.2.2 → 7.2.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (149) hide show
  1. checksums.yaml +5 -5
  2. data/Changes.md +657 -8
  3. data/LICENSE.txt +9 -0
  4. data/README.md +47 -50
  5. data/bin/sidekiq +22 -3
  6. data/bin/sidekiqload +213 -115
  7. data/bin/sidekiqmon +11 -0
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  11. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  12. data/lib/sidekiq/api.rb +558 -353
  13. data/lib/sidekiq/capsule.rb +127 -0
  14. data/lib/sidekiq/cli.rb +238 -260
  15. data/lib/sidekiq/client.rb +127 -102
  16. data/lib/sidekiq/component.rb +68 -0
  17. data/lib/sidekiq/config.rb +287 -0
  18. data/lib/sidekiq/deploy.rb +62 -0
  19. data/lib/sidekiq/embedded.rb +61 -0
  20. data/lib/sidekiq/fetch.rb +49 -42
  21. data/lib/sidekiq/job.rb +374 -0
  22. data/lib/sidekiq/job_logger.rb +35 -9
  23. data/lib/sidekiq/job_retry.rb +162 -102
  24. data/lib/sidekiq/job_util.rb +107 -0
  25. data/lib/sidekiq/launcher.rb +203 -105
  26. data/lib/sidekiq/logger.rb +131 -0
  27. data/lib/sidekiq/manager.rb +43 -46
  28. data/lib/sidekiq/metrics/query.rb +155 -0
  29. data/lib/sidekiq/metrics/shared.rb +95 -0
  30. data/lib/sidekiq/metrics/tracking.rb +136 -0
  31. data/lib/sidekiq/middleware/chain.rb +113 -56
  32. data/lib/sidekiq/middleware/current_attributes.rb +95 -0
  33. data/lib/sidekiq/middleware/i18n.rb +7 -7
  34. data/lib/sidekiq/middleware/modules.rb +21 -0
  35. data/lib/sidekiq/monitor.rb +146 -0
  36. data/lib/sidekiq/paginator.rb +28 -16
  37. data/lib/sidekiq/processor.rb +127 -118
  38. data/lib/sidekiq/rails.rb +50 -39
  39. data/lib/sidekiq/redis_client_adapter.rb +111 -0
  40. data/lib/sidekiq/redis_connection.rb +40 -89
  41. data/lib/sidekiq/ring_buffer.rb +29 -0
  42. data/lib/sidekiq/scheduled.rb +111 -49
  43. data/lib/sidekiq/sd_notify.rb +149 -0
  44. data/lib/sidekiq/systemd.rb +24 -0
  45. data/lib/sidekiq/testing/inline.rb +6 -5
  46. data/lib/sidekiq/testing.rb +90 -89
  47. data/lib/sidekiq/transaction_aware_client.rb +44 -0
  48. data/lib/sidekiq/version.rb +3 -1
  49. data/lib/sidekiq/web/action.rb +15 -11
  50. data/lib/sidekiq/web/application.rb +189 -79
  51. data/lib/sidekiq/web/csrf_protection.rb +180 -0
  52. data/lib/sidekiq/web/helpers.rb +160 -114
  53. data/lib/sidekiq/web/router.rb +23 -19
  54. data/lib/sidekiq/web.rb +68 -107
  55. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  56. data/lib/sidekiq.rb +94 -182
  57. data/sidekiq.gemspec +25 -18
  58. data/web/assets/images/apple-touch-icon.png +0 -0
  59. data/web/assets/javascripts/application.js +146 -61
  60. data/web/assets/javascripts/base-charts.js +106 -0
  61. data/web/assets/javascripts/chart.min.js +13 -0
  62. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  63. data/web/assets/javascripts/dashboard-charts.js +182 -0
  64. data/web/assets/javascripts/dashboard.js +35 -283
  65. data/web/assets/javascripts/metrics.js +298 -0
  66. data/web/assets/stylesheets/application-dark.css +147 -0
  67. data/web/assets/stylesheets/application-rtl.css +2 -95
  68. data/web/assets/stylesheets/application.css +143 -521
  69. data/web/assets/stylesheets/bootstrap.css +1 -1
  70. data/web/locales/ar.yml +71 -65
  71. data/web/locales/cs.yml +62 -62
  72. data/web/locales/da.yml +60 -53
  73. data/web/locales/de.yml +65 -53
  74. data/web/locales/el.yml +43 -24
  75. data/web/locales/en.yml +86 -66
  76. data/web/locales/es.yml +70 -54
  77. data/web/locales/fa.yml +65 -65
  78. data/web/locales/fr.yml +83 -62
  79. data/web/locales/gd.yml +99 -0
  80. data/web/locales/he.yml +65 -64
  81. data/web/locales/hi.yml +59 -59
  82. data/web/locales/it.yml +53 -53
  83. data/web/locales/ja.yml +75 -64
  84. data/web/locales/ko.yml +52 -52
  85. data/web/locales/lt.yml +83 -0
  86. data/web/locales/nb.yml +61 -61
  87. data/web/locales/nl.yml +52 -52
  88. data/web/locales/pl.yml +45 -45
  89. data/web/locales/pt-br.yml +83 -55
  90. data/web/locales/pt.yml +51 -51
  91. data/web/locales/ru.yml +68 -63
  92. data/web/locales/sv.yml +53 -53
  93. data/web/locales/ta.yml +60 -60
  94. data/web/locales/uk.yml +62 -61
  95. data/web/locales/ur.yml +64 -64
  96. data/web/locales/vi.yml +83 -0
  97. data/web/locales/zh-cn.yml +43 -16
  98. data/web/locales/zh-tw.yml +42 -8
  99. data/web/views/_footer.erb +6 -3
  100. data/web/views/_job_info.erb +21 -4
  101. data/web/views/_metrics_period_select.erb +12 -0
  102. data/web/views/_nav.erb +4 -18
  103. data/web/views/_paging.erb +2 -0
  104. data/web/views/_poll_link.erb +3 -6
  105. data/web/views/_summary.erb +7 -7
  106. data/web/views/busy.erb +77 -27
  107. data/web/views/dashboard.erb +48 -18
  108. data/web/views/dead.erb +3 -3
  109. data/web/views/filtering.erb +7 -0
  110. data/web/views/layout.erb +3 -1
  111. data/web/views/metrics.erb +91 -0
  112. data/web/views/metrics_for_job.erb +59 -0
  113. data/web/views/morgue.erb +14 -15
  114. data/web/views/queue.erb +33 -24
  115. data/web/views/queues.erb +19 -5
  116. data/web/views/retries.erb +16 -17
  117. data/web/views/retry.erb +3 -3
  118. data/web/views/scheduled.erb +17 -15
  119. metadata +80 -65
  120. data/.github/contributing.md +0 -32
  121. data/.github/issue_template.md +0 -11
  122. data/.gitignore +0 -13
  123. data/.travis.yml +0 -14
  124. data/3.0-Upgrade.md +0 -70
  125. data/4.0-Upgrade.md +0 -53
  126. data/5.0-Upgrade.md +0 -56
  127. data/COMM-LICENSE +0 -95
  128. data/Ent-Changes.md +0 -221
  129. data/Gemfile +0 -14
  130. data/LICENSE +0 -9
  131. data/Pro-2.0-Upgrade.md +0 -138
  132. data/Pro-3.0-Upgrade.md +0 -44
  133. data/Pro-4.0-Upgrade.md +0 -35
  134. data/Pro-Changes.md +0 -746
  135. data/Rakefile +0 -8
  136. data/bin/sidekiqctl +0 -99
  137. data/code_of_conduct.md +0 -50
  138. data/lib/generators/sidekiq/worker_generator.rb +0 -49
  139. data/lib/sidekiq/core_ext.rb +0 -1
  140. data/lib/sidekiq/delay.rb +0 -42
  141. data/lib/sidekiq/exception_handler.rb +0 -29
  142. data/lib/sidekiq/extensions/action_mailer.rb +0 -57
  143. data/lib/sidekiq/extensions/active_record.rb +0 -40
  144. data/lib/sidekiq/extensions/class_methods.rb +0 -40
  145. data/lib/sidekiq/extensions/generic_proxy.rb +0 -31
  146. data/lib/sidekiq/logging.rb +0 -122
  147. data/lib/sidekiq/middleware/server/active_record.rb +0 -23
  148. data/lib/sidekiq/util.rb +0 -66
  149. data/lib/sidekiq/worker.rb +0 -204
data/LICENSE.txt ADDED
@@ -0,0 +1,9 @@
1
+ Copyright (c) Contributed Systems LLC
2
+
3
+ Sidekiq is an Open Source project licensed under the terms of
4
+ the LGPLv3 license. Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
5
+ for license text.
6
+
7
+ Sidekiq Pro and Sidekiq Enterprise have a commercial-friendly license.
8
+ You can find the commercial license in COMM-LICENSE.txt.
9
+ Please see https://sidekiq.org for purchasing options.
data/README.md CHANGED
@@ -2,10 +2,7 @@ Sidekiq
2
2
  ==============
3
3
 
4
4
  [![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
5
- [![Code Climate](https://codeclimate.com/github/mperham/sidekiq.svg)](https://codeclimate.com/github/mperham/sidekiq)
6
- [![Build Status](https://travis-ci.org/mperham/sidekiq.svg)](https://travis-ci.org/mperham/sidekiq)
7
- [![Gitter Chat](https://badges.gitter.im/mperham/sidekiq.svg)](https://gitter.im/mperham/sidekiq)
8
-
5
+ ![Build](https://github.com/sidekiq/sidekiq/workflows/CI/badge.svg)
9
6
 
10
7
  Simple, efficient background processing for Ruby.
11
8
 
@@ -13,97 +10,97 @@ Sidekiq uses threads to handle many jobs at the same time in the
13
10
  same process. It does not require Rails but will integrate tightly with
14
11
  Rails to make background processing dead simple.
15
12
 
16
- Sidekiq is compatible with Resque. It uses the exact same
17
- message format as Resque so it can integrate into an existing Resque processing farm.
18
- You can have Sidekiq and Resque run side-by-side at the same time and
19
- use the Resque client to enqueue jobs in Redis to be processed by Sidekiq.
20
-
21
- Performance
22
- ---------------
23
-
24
- Version | Latency | Garbage created for 10,000 jobs | Time to process 100,000 jobs | Throughput
25
- -----------------|------|---------|---------|------------------------
26
- Sidekiq 4.0.0 | 10ms | 151 MB | 22 sec | **4500 jobs/sec**
27
- Sidekiq 3.5.1 | 22ms | 1257 MB | 125 sec | 800 jobs/sec
28
- Resque 1.25.2 | - | - | 420 sec | 240 jobs/sec
29
- DelayedJob 4.1.1 | - | - | 465 sec | 215 jobs/sec
30
-
31
- <small>This benchmark can be found in `bin/sidekiqload`.</small>
32
13
 
33
14
  Requirements
34
15
  -----------------
35
16
 
36
- Sidekiq supports CRuby 2.2.2+ and JRuby 9k.
37
-
38
- All Rails releases >= 4.0 are officially supported.
17
+ - Redis: 6.2+
18
+ - Ruby: MRI 2.7+ or JRuby 9.3+.
39
19
 
40
- Redis 2.8 or greater is required. 3.0.3+ is recommended for large
41
- installations with thousands of worker threads.
20
+ Sidekiq 7.0 supports Rails 6.0+ but does not require it.
42
21
 
43
22
 
44
23
  Installation
45
24
  -----------------
46
25
 
47
- gem install sidekiq
26
+ bundle add sidekiq
48
27
 
49
28
 
50
29
  Getting Started
51
30
  -----------------
52
31
 
53
- See the [Getting Started wiki page](https://github.com/mperham/sidekiq/wiki/Getting-Started) and follow the simple setup process.
54
- You can watch [this Youtube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
32
+ See the [Getting Started wiki page](https://github.com/sidekiq/sidekiq/wiki/Getting-Started) and follow the simple setup process.
33
+ You can watch [this YouTube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
55
34
  Sidekiq and see its features in action. Here's the Web UI:
56
35
 
57
- ![Web UI](https://github.com/mperham/sidekiq/raw/master/examples/web-ui.png)
36
+ ![Web UI](https://github.com/sidekiq/sidekiq/raw/main/examples/web-ui.png)
58
37
 
38
+ Performance
39
+ ---------------
40
+
41
+ The benchmark in `bin/sidekiqload` creates 500,000 no-op jobs and drains them as fast as possible, assuming a fixed Redis network latency of 1ms.
42
+ This requires a lot of Redis network I/O and JSON parsing.
43
+ This benchmark is IO-bound so we increase the concurrency to 25.
44
+ If your application is sending lots of emails or performing other network-intensive work, you could see a similar benefit but be careful not to saturate the CPU.
45
+
46
+ Version | Time to process 500k jobs | Throughput (jobs/sec) | Ruby | Concurrency | Job Type
47
+ -----------------|------|---------|---------|------------------------|---
48
+ Sidekiq 7.0.3 | 21.3 sec| 23,500 | 3.2.0+yjit | 30 | Sidekiq::Job
49
+ Sidekiq 7.0.3 | 33.8 sec| 14,700 | 3.2.0+yjit | 30 | ActiveJob 7.0.4
50
+ Sidekiq 7.0.3 | 23.5 sec| 21,300 | 3.2.0 | 30 | Sidekiq::Job
51
+ Sidekiq 7.0.3 | 46.5 sec| 10,700 | 3.2.0 | 30 | ActiveJob 7.0.4
52
+ Sidekiq 7.0.3 | 23.0 sec| 21,700 | 2.7.5 | 30 | Sidekiq::Job
53
+ Sidekiq 7.0.3 | 46.5 sec| 10,850 | 2.7.5 | 30 | ActiveJob 7.0.4
54
+
55
+ Most of Sidekiq's overhead is Redis network I/O.
56
+ ActiveJob adds a notable amount of CPU overhead due to argument deserialization and callbacks.
57
+ Concurrency of 30 was determined experimentally to maximize one CPU without saturating it.
59
58
 
60
59
  Want to Upgrade?
61
60
  -------------------
62
61
 
62
+ Use `bundle up sidekiq` to upgrade Sidekiq and all its dependencies.
63
+ Upgrade notes between each major version can be found in the `docs/` directory.
64
+
63
65
  I also sell Sidekiq Pro and Sidekiq Enterprise, extensions to Sidekiq which provide more
64
66
  features, a commercial-friendly license and allow you to support high
65
67
  quality open source development all at the same time. Please see the
66
- [Sidekiq](http://sidekiq.org/) homepage for more detail.
67
-
68
- Subscribe to the **[quarterly newsletter](https://tinyletter.com/sidekiq)** to stay informed about the latest
69
- features and changes to Sidekiq and its bigger siblings.
68
+ [Sidekiq](https://sidekiq.org/) homepage for more detail.
70
69
 
71
70
 
72
71
  Problems?
73
72
  -----------------
74
73
 
75
- **Please do not directly email any Sidekiq committers with questions or problems.** A community is best served when discussions are held in public.
74
+ **Please do not directly email any Sidekiq committers with questions or problems.**
75
+ A community is best served when discussions are held in public.
76
76
 
77
- If you have a problem, please review the [FAQ](https://github.com/mperham/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/mperham/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages.
78
- Searching the [issues](https://github.com/mperham/sidekiq/issues) for your problem is also a good idea.
77
+ If you have a problem, please review the [FAQ](https://github.com/sidekiq/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/sidekiq/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages.
78
+ Searching the [issues](https://github.com/sidekiq/sidekiq/issues) for your problem is also a good idea.
79
79
 
80
- Sidekiq Pro and Sidekiq Enterprise customers get private email support. You can purchase at http://sidekiq.org; email support@contribsys.com for help.
80
+ Sidekiq Pro and Sidekiq Enterprise customers get private email support.
81
+ You can purchase at https://sidekiq.org; email support@contribsys.com for help.
81
82
 
82
83
  Useful resources:
83
84
 
84
- * Product documentation is in the [wiki](https://github.com/mperham/sidekiq/wiki).
85
- * Release announcements are made to the [@sidekiq](https://twitter.com/sidekiq) Twitter account.
85
+ * Product documentation is in the [wiki](https://github.com/sidekiq/sidekiq/wiki).
86
+ * Occasional announcements are made to the [@sidekiq](https://ruby.social/@sidekiq) Mastodon account.
86
87
  * The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q &amp; A.
87
88
 
88
- **No support via Twitter**
89
-
90
- Every Friday morning is Sidekiq happy hour: I video chat and answer questions.
91
- See the [Sidekiq support page](http://sidekiq.org/support.html) for details.
89
+ Every Friday morning is Sidekiq office hour: I video chat and answer questions.
90
+ See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
92
91
 
93
- Thanks
92
+ Contributing
94
93
  -----------------
95
94
 
96
- Sidekiq stays fast by using the [JProfiler java profiler](http://www.ej-technologies.com/products/jprofiler/overview.html) to find and fix
97
- performance problems on JRuby. Unfortunately MRI does not have good multithreaded profiling tools.
98
-
95
+ Please see [the contributing guidelines](https://github.com/sidekiq/sidekiq/blob/main/.github/contributing.md).
99
96
 
100
97
  License
101
98
  -----------------
102
99
 
103
- Please see [LICENSE](https://github.com/mperham/sidekiq/blob/master/LICENSE) for licensing details.
104
-
100
+ Please see [LICENSE.txt](https://github.com/sidekiq/sidekiq/blob/main/LICENSE.txt) for licensing details.
101
+ The license for Sidekiq Pro and Sidekiq Enterprise can be found in [COMM-LICENSE.txt](https://github.com/sidekiq/sidekiq/blob/main/COMM-LICENSE.txt).
105
102
 
106
103
  Author
107
104
  -----------------
108
105
 
109
- Mike Perham, [@mperham](https://twitter.com/mperham) / [@sidekiq](https://twitter.com/sidekiq), [http://www.mikeperham.com](http://www.mikeperham.com) / [http://www.contribsys.com](http://www.contribsys.com)
106
+ Mike Perham, [@getajobmike](https://ruby.social/@getajobmike) / [@sidekiq](https://ruby.social/@sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
data/bin/sidekiq CHANGED
@@ -4,15 +4,34 @@
4
4
  # RUBYOPT=-w bundle exec sidekiq
5
5
  $TESTING = false
6
6
 
7
- require_relative '../lib/sidekiq/cli'
7
+ require_relative "../lib/sidekiq/cli"
8
+
9
+ def integrate_with_systemd
10
+ return unless ENV["NOTIFY_SOCKET"]
11
+
12
+ Sidekiq.configure_server do |config|
13
+ config.logger.info "Enabling systemd notification integration"
14
+ require "sidekiq/sd_notify"
15
+ config.on(:startup) do
16
+ Sidekiq::SdNotify.ready
17
+ end
18
+ config.on(:shutdown) do
19
+ Sidekiq::SdNotify.stopping
20
+ end
21
+ Sidekiq.start_watchdog if Sidekiq::SdNotify.watchdog?
22
+ end
23
+ end
8
24
 
9
25
  begin
10
26
  cli = Sidekiq::CLI.instance
11
27
  cli.parse
28
+
29
+ integrate_with_systemd
30
+
12
31
  cli.run
13
32
  rescue => e
14
33
  raise e if $DEBUG
15
- STDERR.puts e.message
16
- STDERR.puts e.backtrace.join("\n")
34
+ warn e.message
35
+ warn e.backtrace.join("\n")
17
36
  exit 1
18
37
  end
data/bin/sidekiqload CHANGED
@@ -1,149 +1,247 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
+ #
4
+ # bin/sidekiqload is a helpful script to load test and
5
+ # performance tune Sidekiq's core. It creates 500,000 no-op
6
+ # jobs and executes them as fast as possible.
7
+ # Example Usage:
8
+ #
9
+ # > RUBY_YJIT_ENABLE=1 LATENCY=0 THREADS=10 bin/sidekiqload
10
+ # Result: Done, 500000 jobs in 20.264945 sec, 24673 jobs/sec
11
+ #
12
+ # Use LATENCY=1 to get a more real world network setup
13
+ # but you'll need to setup and start toxiproxy as noted below.
14
+ #
15
+ # Use AJ=1 to test ActiveJob instead of plain old Sidekiq::Jobs so
16
+ # you can see the runtime performance difference between the two APIs.
17
+ #
18
+ # None of this script is considered a public API and may change over time.
19
+ #
20
+
3
21
  # Quiet some warnings we see when running in warning mode:
4
22
  # RUBYOPT=-w bundle exec sidekiq
5
23
  $TESTING = false
24
+ puts RUBY_DESCRIPTION
25
+ puts(%w[THREADS LATENCY AJ PROFILE].map { |x| "#{x}: #{ENV[x] || "nil"}" }.join(", "))
26
+
27
+ require "ruby-prof" if ENV["PROFILE"]
28
+ require "bundler/setup"
29
+ Bundler.require(:default, :load_test)
30
+
31
+ latency = Integer(ENV["LATENCY"] || 1)
32
+ if latency > 0
33
+ # brew tap shopify/shopify
34
+ # brew install toxiproxy
35
+ # run `toxiproxy-server` in a separate terminal window.
36
+ require "toxiproxy"
37
+ # simulate a non-localhost network for realer-world conditions.
38
+ # adding 1ms of network latency has an ENORMOUS impact on benchmarks
39
+ Toxiproxy.populate([{
40
+ name: "redis",
41
+ listen: "127.0.0.1:6380",
42
+ upstream: "127.0.0.1:6379"
43
+ }])
44
+ end
6
45
 
7
- #require 'ruby-prof'
8
- Bundler.require(:default)
9
-
10
- require_relative '../lib/sidekiq/cli'
11
- require_relative '../lib/sidekiq/launcher'
12
-
13
- include Sidekiq::Util
46
+ if ENV["AJ"]
47
+ require "active_job"
48
+ puts "Using ActiveJob #{ActiveJob::VERSION::STRING}"
49
+ ActiveJob::Base.queue_adapter = :sidekiq
50
+ ActiveJob::Base.logger.level = Logger::WARN
14
51
 
15
- Sidekiq.configure_server do |config|
16
- #config.options[:concurrency] = 1
17
- config.redis = { db: 13 }
18
- config.options[:queues] << 'default'
19
- config.logger.level = Logger::ERROR
20
- config.average_scheduled_poll_interval = 2
21
- config.reliable! if defined?(Sidekiq::Pro)
52
+ class LoadJob < ActiveJob::Base
53
+ def perform(idx, ts = nil)
54
+ puts(Time.now.to_f - ts) if !ts.nil?
55
+ end
56
+ end
22
57
  end
23
58
 
24
59
  class LoadWorker
25
- include Sidekiq::Worker
60
+ include Sidekiq::Job
26
61
  sidekiq_options retry: 1
27
62
  sidekiq_retry_in do |x|
28
63
  1
29
64
  end
30
65
 
31
- def perform(idx)
32
- #raise idx.to_s if idx % 100 == 1
66
+ def perform(idx, ts = nil)
67
+ puts(Time.now.to_f - ts) if !ts.nil?
68
+ # raise idx.to_s if idx % 100 == 1
33
69
  end
34
70
  end
35
71
 
36
- # brew tap shopify/shopify
37
- # brew install toxiproxy
38
- # gem install toxiproxy
39
- #require 'toxiproxy'
40
- # simulate a non-localhost network for realer-world conditions.
41
- # adding 1ms of network latency has an ENORMOUS impact on benchmarks
42
- #Toxiproxy.populate([{
43
- #"name": "redis",
44
- #"listen": "127.0.0.1:6380",
45
- #"upstream": "127.0.0.1:6379"
46
- #}])
47
-
48
- self_read, self_write = IO.pipe
49
- %w(INT TERM TSTP TTIN).each do |sig|
50
- begin
51
- trap sig do
52
- self_write.puts(sig)
53
- end
54
- rescue ArgumentError
55
- puts "Signal #{sig} not supported"
56
- end
72
+ def Process.rss
73
+ `ps -o rss= -p #{Process.pid}`.chomp.to_i
57
74
  end
58
75
 
59
- Sidekiq.redis {|c| c.flushdb}
60
- def handle_signal(launcher, sig)
61
- Sidekiq.logger.debug "Got #{sig} signal"
62
- case sig
63
- when 'INT'
64
- # Handle Ctrl-C in JRuby like MRI
65
- # http://jira.codehaus.org/browse/JRUBY-4637
66
- raise Interrupt
67
- when 'TERM'
68
- # Heroku sends TERM and then waits 10 seconds for process to exit.
69
- raise Interrupt
70
- when 'TSTP'
71
- Sidekiq.logger.info "Received TSTP, no longer accepting new work"
72
- launcher.quiet
73
- when 'TTIN'
74
- Thread.list.each do |thread|
75
- Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['label']}"
76
- if thread.backtrace
77
- Sidekiq.logger.warn thread.backtrace.join("\n")
78
- else
79
- Sidekiq.logger.warn "<no backtrace available>"
80
- end
81
- end
76
+ class Loader
77
+ def initialize
78
+ @iter = ENV["GC"] ? 10 : 500
79
+ @count = Integer(ENV["COUNT"] || 1_000)
80
+ @latency = Integer(ENV["LATENCY"] || 1)
82
81
  end
83
- end
84
82
 
85
- def Process.rss
86
- `ps -o rss= -p #{Process.pid}`.chomp.to_i
87
- end
83
+ def configure
84
+ @x = Sidekiq.configure_embed do |config|
85
+ config.redis = {db: 13, port: ((@latency > 0) ? 6380 : 6379)}
86
+ config.concurrency = Integer(ENV.fetch("THREADS", "10"))
87
+ # config.redis = { db: 13, port: 6380, driver: :hiredis}
88
+ config.queues = %w[default]
89
+ config.logger.level = Logger::WARN
90
+ config.average_scheduled_poll_interval = 2
91
+ config.reliable! if defined?(Sidekiq::Pro)
92
+ end
88
93
 
89
- iter = 10
90
- count = 10_000
94
+ @self_read, @self_write = IO.pipe
95
+ %w[INT TERM TSTP TTIN].each do |sig|
96
+ trap sig do
97
+ @self_write.puts(sig)
98
+ end
99
+ rescue ArgumentError
100
+ puts "Signal #{sig} not supported"
101
+ end
102
+ end
91
103
 
92
- iter.times do
93
- arr = Array.new(count) do
94
- []
104
+ def handle_signal(sig)
105
+ launcher = @x
106
+ Sidekiq.logger.debug "Got #{sig} signal"
107
+ case sig
108
+ when "INT"
109
+ # Handle Ctrl-C in JRuby like MRI
110
+ # http://jira.codehaus.org/browse/JRUBY-4637
111
+ raise Interrupt
112
+ when "TERM"
113
+ # Heroku sends TERM and then waits 30 seconds for process to exit.
114
+ raise Interrupt
115
+ when "TSTP"
116
+ Sidekiq.logger.info "Received TSTP, no longer accepting new work"
117
+ launcher.quiet
118
+ when "TTIN"
119
+ Thread.list.each do |thread|
120
+ Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["label"]}"
121
+ if thread.backtrace
122
+ Sidekiq.logger.warn thread.backtrace.join("\n")
123
+ else
124
+ Sidekiq.logger.warn "<no backtrace available>"
125
+ end
126
+ end
127
+ end
95
128
  end
96
- count.times do |idx|
97
- arr[idx][0] = idx
129
+
130
+ def setup
131
+ Sidekiq.logger.error("Setup RSS: #{Process.rss}")
132
+ Sidekiq.redis { |c| c.flushdb }
133
+ start = Time.now
134
+ if ENV["AJ"]
135
+ @iter.times do
136
+ @count.times do |idx|
137
+ LoadJob.perform_later(idx)
138
+ end
139
+ end
140
+ else
141
+ @iter.times do
142
+ arr = Array.new(@count) { |idx| [idx] }
143
+ Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
144
+ end
145
+ end
146
+ Sidekiq.logger.warn "Created #{@count * @iter} jobs in #{Time.now - start} sec"
98
147
  end
99
- Sidekiq::Client.push_bulk('class' => LoadWorker, 'args' => arr)
100
- end
101
- Sidekiq.logger.error "Created #{count*iter} jobs"
102
-
103
- Monitoring = Thread.new do
104
- watchdog("monitor thread") do
105
- while true
106
- sleep 1
107
- qsize, retries = Sidekiq.redis do |conn|
108
- conn.pipelined do
148
+
149
+ def monitor
150
+ @monitor = Thread.new do
151
+ GC.start
152
+ loop do
153
+ sleep 0.2
154
+ qsize = Sidekiq.redis do |conn|
109
155
  conn.llen "queue:default"
110
- conn.zcard "retry"
111
156
  end
112
- end.map(&:to_i)
113
- total = qsize + retries
114
- #GC.start
115
- Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
116
- if total == 0
117
- Sidekiq.logger.error("Done")
118
- exit(0)
157
+ total = qsize
158
+ if total == 0
159
+ ending = Time.now - @start
160
+ size = @iter * @count
161
+ Sidekiq.logger.error("Done, #{size} jobs in #{ending} sec, #{(size / ending).to_i} jobs/sec")
162
+ Sidekiq.logger.error("Ending RSS: #{Process.rss}")
163
+ Sidekiq.logger.error("Now here's the latency for three jobs")
164
+
165
+ if ENV["AJ"]
166
+ LoadJob.perform_later(1, Time.now.to_f)
167
+ LoadJob.perform_later(2, Time.now.to_f)
168
+ LoadJob.perform_later(3, Time.now.to_f)
169
+ else
170
+ LoadWorker.perform_async(1, Time.now.to_f)
171
+ LoadWorker.perform_async(2, Time.now.to_f)
172
+ LoadWorker.perform_async(3, Time.now.to_f)
173
+ end
174
+
175
+ sleep 0.1
176
+ @x.stop
177
+ Process.kill("INT", $$)
178
+ break
179
+ end
119
180
  end
120
181
  end
121
182
  end
122
- end
123
183
 
124
- begin
125
- #RubyProf::exclude_threads = [ Monitoring ]
126
- #RubyProf.start
127
- fire_event(:startup)
128
- #Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
129
- #Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
130
- launcher = Sidekiq::Launcher.new(Sidekiq.options)
131
- launcher.run
132
-
133
- while readable_io = IO.select([self_read])
134
- signal = readable_io.first[0].gets.strip
135
- handle_signal(launcher, signal)
184
+ def with_latency(latency, &block)
185
+ Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
186
+ if latency > 0
187
+ Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
188
+ else
189
+ yield
190
+ end
191
+ end
192
+
193
+ def run(name)
194
+ Sidekiq.logger.warn("Starting #{name}")
195
+ monitor
196
+
197
+ if ENV["PROFILE"]
198
+ RubyProf.exclude_threads = [@monitor]
199
+ RubyProf.start
200
+ elsif ENV["GC"]
201
+ GC.start
202
+ GC.compact
203
+ GC.disable
204
+ Sidekiq.logger.error("GC Start RSS: #{Process.rss}")
136
205
  end
137
- #end
138
- rescue SystemExit => e
139
- #Sidekiq.logger.error("Profiling...")
140
- #result = RubyProf.stop
141
- #printer = RubyProf::GraphHtmlPrinter.new(result)
142
- #printer.print(File.new("output.html", "w"), :min_percent => 1)
143
- # normal
144
- rescue => e
145
- raise e if $DEBUG
146
- STDERR.puts e.message
147
- STDERR.puts e.backtrace.join("\n")
148
- exit 1
206
+ @start = Time.now
207
+ with_latency(@latency) do
208
+ @x.run
209
+
210
+ while (readable_io = IO.select([@self_read]))
211
+ signal = readable_io.first[0].gets.strip
212
+ handle_signal(signal)
213
+ end
214
+ end
215
+ # normal
216
+ rescue Interrupt
217
+ rescue => e
218
+ raise e if $DEBUG
219
+ warn e.message
220
+ warn e.backtrace.join("\n")
221
+ exit 1
222
+ ensure
223
+ @x.stop
224
+ end
225
+
226
+ def done
227
+ Sidekiq.logger.error("GC End RSS: #{Process.rss}") if ENV["GC"]
228
+ if ENV["PROFILE"]
229
+ Sidekiq.logger.error("Profiling...")
230
+ result = RubyProf.stop
231
+ printer = RubyProf::GraphHtmlPrinter.new(result)
232
+ printer.print(File.new("output.html", "w"), min_percent: 1)
233
+ end
234
+ end
149
235
  end
236
+
237
+ ll = Loader.new
238
+ ll.configure
239
+
240
+ if ENV["WARM"]
241
+ ll.setup
242
+ ll.run("warmup")
243
+ end
244
+
245
+ ll.setup
246
+ ll.run("load")
247
+ ll.done
data/bin/sidekiqmon ADDED
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "sidekiq/monitor"
4
+
5
+ # disable the Redis connection pool logging
6
+ Sidekiq.default_configuration.logger.level = :warn
7
+
8
+ section = "all"
9
+ section = ARGV[0] if ARGV.size == 1
10
+
11
+ Sidekiq::Monitor::Status.new.display(section)
@@ -0,0 +1,57 @@
1
+ require "rails/generators/named_base"
2
+
3
+ module Sidekiq
4
+ module Generators # :nodoc:
5
+ class JobGenerator < ::Rails::Generators::NamedBase # :nodoc:
6
+ desc "This generator creates a Sidekiq Job in app/sidekiq and a corresponding test"
7
+
8
+ check_class_collision suffix: "Job"
9
+
10
+ def self.default_generator_root
11
+ File.dirname(__FILE__)
12
+ end
13
+
14
+ def create_job_file
15
+ template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
16
+ end
17
+
18
+ def create_test_file
19
+ return unless test_framework
20
+
21
+ if test_framework == :rspec
22
+ create_job_spec
23
+ else
24
+ create_job_test
25
+ end
26
+ end
27
+
28
+ private
29
+
30
+ def create_job_spec
31
+ template_file = File.join(
32
+ "spec/sidekiq",
33
+ class_path,
34
+ "#{file_name}_job_spec.rb"
35
+ )
36
+ template "job_spec.rb.erb", template_file
37
+ end
38
+
39
+ def create_job_test
40
+ template_file = File.join(
41
+ "test/sidekiq",
42
+ class_path,
43
+ "#{file_name}_job_test.rb"
44
+ )
45
+ template "job_test.rb.erb", template_file
46
+ end
47
+
48
+ def file_name
49
+ @_file_name ||= super.sub(/_?job\z/i, "")
50
+ end
51
+
52
+ def test_framework
53
+ ::Rails.application.config.generators.options[:rails][:test_framework]
54
+ end
55
+ end
56
+ end
57
+ end
@@ -1,6 +1,6 @@
1
1
  <% module_namespacing do -%>
2
- class <%= class_name %>Worker
3
- include Sidekiq::Worker
2
+ class <%= class_name %>Job
3
+ include Sidekiq::Job
4
4
 
5
5
  def perform(*args)
6
6
  # Do something
@@ -1,6 +1,6 @@
1
1
  require 'rails_helper'
2
2
  <% module_namespacing do -%>
3
- RSpec.describe <%= class_name %>Worker, type: :worker do
3
+ RSpec.describe <%= class_name %>Job, type: :job do
4
4
  pending "add some examples to (or delete) #{__FILE__}"
5
5
  end
6
6
  <% end -%>
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
  <% module_namespacing do -%>
3
- class <%= class_name %>WorkerTest < <% if defined? Minitest::Test %>Minitest::Test<% else %>MiniTest::Unit::TestCase<% end %>
3
+ class <%= class_name %>JobTest < Minitest::Test
4
4
  def test_example
5
5
  skip "add some examples to (or delete) #{__FILE__}"
6
6
  end