sidekiq 3.5.4 → 7.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. checksums.yaml +5 -5
  2. data/Changes.md +992 -6
  3. data/LICENSE.txt +9 -0
  4. data/README.md +52 -43
  5. data/bin/sidekiq +22 -4
  6. data/bin/sidekiqload +209 -115
  7. data/bin/sidekiqmon +11 -0
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/job_spec.rb.erb +6 -0
  11. data/lib/generators/sidekiq/templates/job_test.rb.erb +8 -0
  12. data/lib/sidekiq/api.rb +633 -295
  13. data/lib/sidekiq/capsule.rb +127 -0
  14. data/lib/sidekiq/cli.rb +270 -248
  15. data/lib/sidekiq/client.rb +139 -108
  16. data/lib/sidekiq/component.rb +68 -0
  17. data/lib/sidekiq/config.rb +287 -0
  18. data/lib/sidekiq/deploy.rb +62 -0
  19. data/lib/sidekiq/embedded.rb +61 -0
  20. data/lib/sidekiq/fetch.rb +53 -121
  21. data/lib/sidekiq/job.rb +374 -0
  22. data/lib/sidekiq/job_logger.rb +51 -0
  23. data/lib/sidekiq/job_retry.rb +301 -0
  24. data/lib/sidekiq/job_util.rb +107 -0
  25. data/lib/sidekiq/launcher.rb +241 -69
  26. data/lib/sidekiq/logger.rb +131 -0
  27. data/lib/sidekiq/manager.rb +88 -190
  28. data/lib/sidekiq/metrics/query.rb +155 -0
  29. data/lib/sidekiq/metrics/shared.rb +95 -0
  30. data/lib/sidekiq/metrics/tracking.rb +136 -0
  31. data/lib/sidekiq/middleware/chain.rb +114 -56
  32. data/lib/sidekiq/middleware/current_attributes.rb +95 -0
  33. data/lib/sidekiq/middleware/i18n.rb +8 -7
  34. data/lib/sidekiq/middleware/modules.rb +21 -0
  35. data/lib/sidekiq/monitor.rb +146 -0
  36. data/lib/sidekiq/paginator.rb +29 -16
  37. data/lib/sidekiq/processor.rb +238 -118
  38. data/lib/sidekiq/rails.rb +57 -27
  39. data/lib/sidekiq/redis_client_adapter.rb +111 -0
  40. data/lib/sidekiq/redis_connection.rb +49 -50
  41. data/lib/sidekiq/ring_buffer.rb +29 -0
  42. data/lib/sidekiq/scheduled.rb +173 -52
  43. data/lib/sidekiq/sd_notify.rb +149 -0
  44. data/lib/sidekiq/systemd.rb +24 -0
  45. data/lib/sidekiq/testing/inline.rb +7 -5
  46. data/lib/sidekiq/testing.rb +197 -65
  47. data/lib/sidekiq/transaction_aware_client.rb +44 -0
  48. data/lib/sidekiq/version.rb +4 -1
  49. data/lib/sidekiq/web/action.rb +93 -0
  50. data/lib/sidekiq/web/application.rb +463 -0
  51. data/lib/sidekiq/web/csrf_protection.rb +180 -0
  52. data/lib/sidekiq/web/helpers.rb +364 -0
  53. data/lib/sidekiq/web/router.rb +104 -0
  54. data/lib/sidekiq/web.rb +113 -216
  55. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  56. data/lib/sidekiq.rb +99 -142
  57. data/sidekiq.gemspec +26 -23
  58. data/web/assets/images/apple-touch-icon.png +0 -0
  59. data/web/assets/javascripts/application.js +163 -74
  60. data/web/assets/javascripts/base-charts.js +106 -0
  61. data/web/assets/javascripts/chart.min.js +13 -0
  62. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  63. data/web/assets/javascripts/dashboard-charts.js +182 -0
  64. data/web/assets/javascripts/dashboard.js +37 -280
  65. data/web/assets/javascripts/metrics.js +298 -0
  66. data/web/assets/stylesheets/application-dark.css +147 -0
  67. data/web/assets/stylesheets/application-rtl.css +153 -0
  68. data/web/assets/stylesheets/application.css +181 -198
  69. data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
  70. data/web/assets/stylesheets/bootstrap.css +4 -8
  71. data/web/locales/ar.yml +87 -0
  72. data/web/locales/cs.yml +62 -52
  73. data/web/locales/da.yml +60 -53
  74. data/web/locales/de.yml +65 -53
  75. data/web/locales/el.yml +43 -24
  76. data/web/locales/en.yml +86 -62
  77. data/web/locales/es.yml +70 -53
  78. data/web/locales/fa.yml +80 -0
  79. data/web/locales/fr.yml +86 -56
  80. data/web/locales/gd.yml +99 -0
  81. data/web/locales/he.yml +80 -0
  82. data/web/locales/hi.yml +59 -59
  83. data/web/locales/it.yml +53 -53
  84. data/web/locales/ja.yml +78 -56
  85. data/web/locales/ko.yml +52 -52
  86. data/web/locales/lt.yml +83 -0
  87. data/web/locales/nb.yml +61 -61
  88. data/web/locales/nl.yml +52 -52
  89. data/web/locales/pl.yml +45 -45
  90. data/web/locales/pt-br.yml +83 -55
  91. data/web/locales/pt.yml +51 -51
  92. data/web/locales/ru.yml +68 -60
  93. data/web/locales/sv.yml +53 -53
  94. data/web/locales/ta.yml +60 -60
  95. data/web/locales/uk.yml +62 -61
  96. data/web/locales/ur.yml +80 -0
  97. data/web/locales/vi.yml +83 -0
  98. data/web/locales/zh-cn.yml +43 -16
  99. data/web/locales/zh-tw.yml +42 -8
  100. data/web/views/_footer.erb +10 -9
  101. data/web/views/_job_info.erb +26 -5
  102. data/web/views/_metrics_period_select.erb +12 -0
  103. data/web/views/_nav.erb +6 -20
  104. data/web/views/_paging.erb +3 -1
  105. data/web/views/_poll_link.erb +3 -6
  106. data/web/views/_summary.erb +7 -7
  107. data/web/views/busy.erb +87 -28
  108. data/web/views/dashboard.erb +51 -21
  109. data/web/views/dead.erb +4 -4
  110. data/web/views/filtering.erb +7 -0
  111. data/web/views/layout.erb +15 -5
  112. data/web/views/metrics.erb +91 -0
  113. data/web/views/metrics_for_job.erb +59 -0
  114. data/web/views/morgue.erb +25 -22
  115. data/web/views/queue.erb +35 -25
  116. data/web/views/queues.erb +23 -7
  117. data/web/views/retries.erb +28 -23
  118. data/web/views/retry.erb +5 -5
  119. data/web/views/scheduled.erb +19 -17
  120. data/web/views/scheduled_job_info.erb +1 -1
  121. metadata +86 -268
  122. data/.gitignore +0 -12
  123. data/.travis.yml +0 -16
  124. data/3.0-Upgrade.md +0 -70
  125. data/COMM-LICENSE +0 -95
  126. data/Contributing.md +0 -32
  127. data/Ent-Changes.md +0 -39
  128. data/Gemfile +0 -27
  129. data/LICENSE +0 -9
  130. data/Pro-2.0-Upgrade.md +0 -138
  131. data/Pro-Changes.md +0 -454
  132. data/Rakefile +0 -9
  133. data/bin/sidekiqctl +0 -93
  134. data/lib/generators/sidekiq/templates/worker_spec.rb.erb +0 -6
  135. data/lib/generators/sidekiq/templates/worker_test.rb.erb +0 -8
  136. data/lib/generators/sidekiq/worker_generator.rb +0 -49
  137. data/lib/sidekiq/actor.rb +0 -39
  138. data/lib/sidekiq/core_ext.rb +0 -105
  139. data/lib/sidekiq/exception_handler.rb +0 -30
  140. data/lib/sidekiq/extensions/action_mailer.rb +0 -56
  141. data/lib/sidekiq/extensions/active_record.rb +0 -39
  142. data/lib/sidekiq/extensions/class_methods.rb +0 -39
  143. data/lib/sidekiq/extensions/generic_proxy.rb +0 -24
  144. data/lib/sidekiq/logging.rb +0 -104
  145. data/lib/sidekiq/middleware/server/active_record.rb +0 -13
  146. data/lib/sidekiq/middleware/server/logging.rb +0 -40
  147. data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -206
  148. data/lib/sidekiq/util.rb +0 -68
  149. data/lib/sidekiq/web_helpers.rb +0 -249
  150. data/lib/sidekiq/worker.rb +0 -103
  151. data/test/config.yml +0 -9
  152. data/test/env_based_config.yml +0 -11
  153. data/test/fake_env.rb +0 -0
  154. data/test/fixtures/en.yml +0 -2
  155. data/test/helper.rb +0 -49
  156. data/test/test_api.rb +0 -493
  157. data/test/test_cli.rb +0 -335
  158. data/test/test_client.rb +0 -194
  159. data/test/test_exception_handler.rb +0 -55
  160. data/test/test_extensions.rb +0 -126
  161. data/test/test_fetch.rb +0 -104
  162. data/test/test_logging.rb +0 -34
  163. data/test/test_manager.rb +0 -168
  164. data/test/test_middleware.rb +0 -159
  165. data/test/test_processor.rb +0 -237
  166. data/test/test_rails.rb +0 -21
  167. data/test/test_redis_connection.rb +0 -126
  168. data/test/test_retry.rb +0 -325
  169. data/test/test_scheduled.rb +0 -114
  170. data/test/test_scheduling.rb +0 -49
  171. data/test/test_sidekiq.rb +0 -99
  172. data/test/test_testing.rb +0 -142
  173. data/test/test_testing_fake.rb +0 -268
  174. data/test/test_testing_inline.rb +0 -93
  175. data/test/test_util.rb +0 -16
  176. data/test/test_web.rb +0 -608
  177. data/test/test_web_helpers.rb +0 -53
  178. data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
  179. data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
  180. data/web/assets/images/status/active.png +0 -0
  181. data/web/assets/images/status/idle.png +0 -0
  182. data/web/assets/javascripts/locales/README.md +0 -27
  183. data/web/assets/javascripts/locales/jquery.timeago.ar.js +0 -96
  184. data/web/assets/javascripts/locales/jquery.timeago.bg.js +0 -18
  185. data/web/assets/javascripts/locales/jquery.timeago.bs.js +0 -49
  186. data/web/assets/javascripts/locales/jquery.timeago.ca.js +0 -18
  187. data/web/assets/javascripts/locales/jquery.timeago.cs.js +0 -18
  188. data/web/assets/javascripts/locales/jquery.timeago.cy.js +0 -20
  189. data/web/assets/javascripts/locales/jquery.timeago.da.js +0 -18
  190. data/web/assets/javascripts/locales/jquery.timeago.de.js +0 -18
  191. data/web/assets/javascripts/locales/jquery.timeago.el.js +0 -18
  192. data/web/assets/javascripts/locales/jquery.timeago.en-short.js +0 -20
  193. data/web/assets/javascripts/locales/jquery.timeago.en.js +0 -20
  194. data/web/assets/javascripts/locales/jquery.timeago.es.js +0 -18
  195. data/web/assets/javascripts/locales/jquery.timeago.et.js +0 -18
  196. data/web/assets/javascripts/locales/jquery.timeago.fa.js +0 -22
  197. data/web/assets/javascripts/locales/jquery.timeago.fi.js +0 -28
  198. data/web/assets/javascripts/locales/jquery.timeago.fr-short.js +0 -16
  199. data/web/assets/javascripts/locales/jquery.timeago.fr.js +0 -17
  200. data/web/assets/javascripts/locales/jquery.timeago.he.js +0 -18
  201. data/web/assets/javascripts/locales/jquery.timeago.hr.js +0 -49
  202. data/web/assets/javascripts/locales/jquery.timeago.hu.js +0 -18
  203. data/web/assets/javascripts/locales/jquery.timeago.hy.js +0 -18
  204. data/web/assets/javascripts/locales/jquery.timeago.id.js +0 -18
  205. data/web/assets/javascripts/locales/jquery.timeago.it.js +0 -16
  206. data/web/assets/javascripts/locales/jquery.timeago.ja.js +0 -19
  207. data/web/assets/javascripts/locales/jquery.timeago.ko.js +0 -17
  208. data/web/assets/javascripts/locales/jquery.timeago.lt.js +0 -20
  209. data/web/assets/javascripts/locales/jquery.timeago.mk.js +0 -20
  210. data/web/assets/javascripts/locales/jquery.timeago.nl.js +0 -20
  211. data/web/assets/javascripts/locales/jquery.timeago.no.js +0 -18
  212. data/web/assets/javascripts/locales/jquery.timeago.pl.js +0 -31
  213. data/web/assets/javascripts/locales/jquery.timeago.pt-br.js +0 -16
  214. data/web/assets/javascripts/locales/jquery.timeago.pt.js +0 -16
  215. data/web/assets/javascripts/locales/jquery.timeago.ro.js +0 -18
  216. data/web/assets/javascripts/locales/jquery.timeago.rs.js +0 -49
  217. data/web/assets/javascripts/locales/jquery.timeago.ru.js +0 -34
  218. data/web/assets/javascripts/locales/jquery.timeago.sk.js +0 -18
  219. data/web/assets/javascripts/locales/jquery.timeago.sl.js +0 -44
  220. data/web/assets/javascripts/locales/jquery.timeago.sv.js +0 -18
  221. data/web/assets/javascripts/locales/jquery.timeago.th.js +0 -20
  222. data/web/assets/javascripts/locales/jquery.timeago.tr.js +0 -16
  223. data/web/assets/javascripts/locales/jquery.timeago.uk.js +0 -34
  224. data/web/assets/javascripts/locales/jquery.timeago.uz.js +0 -19
  225. data/web/assets/javascripts/locales/jquery.timeago.zh-cn.js +0 -20
  226. data/web/assets/javascripts/locales/jquery.timeago.zh-tw.js +0 -20
  227. data/web/views/_poll_js.erb +0 -5
  228. /data/web/assets/images/{status-sd8051fd480.png → status.png} +0 -0
data/LICENSE.txt ADDED
@@ -0,0 +1,9 @@
1
+ Copyright (c) Contributed Systems LLC
2
+
3
+ Sidekiq is an Open Source project licensed under the terms of
4
+ the LGPLv3 license. Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
5
+ for license text.
6
+
7
+ Sidekiq Pro and Sidekiq Enterprise have a commercial-friendly license.
8
+ You can find the commercial license in COMM-LICENSE.txt.
9
+ Please see https://sidekiq.org for purchasing options.
data/README.md CHANGED
@@ -2,96 +2,105 @@ Sidekiq
2
2
  ==============
3
3
 
4
4
  [![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
5
- [![Code Climate](https://codeclimate.com/github/mperham/sidekiq.svg)](https://codeclimate.com/github/mperham/sidekiq)
6
- [![Build Status](https://travis-ci.org/mperham/sidekiq.svg)](https://travis-ci.org/mperham/sidekiq)
7
- [![Gitter Chat](https://badges.gitter.im/mperham/sidekiq.svg)](https://gitter.im/mperham/sidekiq)
8
-
5
+ ![Build](https://github.com/sidekiq/sidekiq/workflows/CI/badge.svg)
9
6
 
10
7
  Simple, efficient background processing for Ruby.
11
8
 
12
9
  Sidekiq uses threads to handle many jobs at the same time in the
13
10
  same process. It does not require Rails but will integrate tightly with
14
- Rails 3/4 to make background processing dead simple.
15
-
16
- Sidekiq is compatible with Resque. It uses the exact same
17
- message format as Resque so it can integrate into an existing Resque processing farm.
18
- You can have Sidekiq and Resque run side-by-side at the same time and
19
- use the Resque client to enqueue jobs in Redis to be processed by Sidekiq.
20
-
21
- At the same time, Sidekiq uses multithreading so it is much more memory efficient than
22
- Resque (which forks a new process for every job). You'll find that you might need
23
- 10 200MB resque processes to peg your CPU whereas one 300MB Sidekiq process will peg
24
- the same CPU and perform the same amount of work.
11
+ Rails to make background processing dead simple.
25
12
 
26
13
 
27
14
  Requirements
28
15
  -----------------
29
16
 
30
- I test with the latest MRI (2.2, 2.1 and 2.0) and JRuby versions (1.7). Other versions/VMs
31
- are untested but might work fine. MRI 1.9 is no longer supported.
32
-
33
- All Rails releases starting from 3.2 are officially supported.
17
+ - Redis: 6.2+
18
+ - Ruby: MRI 2.7+ or JRuby 9.3+.
34
19
 
35
- Redis 2.4 or greater is required.
20
+ Sidekiq 7.0 supports Rails 6.0+ but does not require it.
36
21
 
37
22
 
38
23
  Installation
39
24
  -----------------
40
25
 
41
- gem install sidekiq
26
+ bundle add sidekiq
42
27
 
43
28
 
44
29
  Getting Started
45
30
  -----------------
46
31
 
47
- See the [Getting Started wiki page](https://github.com/mperham/sidekiq/wiki/Getting-Started) and follow the simple setup process.
48
- You can watch [Railscast #366](http://railscasts.com/episodes/366-sidekiq) to see Sidekiq in action. If you do everything right, you should see this:
32
+ See the [Getting Started wiki page](https://github.com/sidekiq/sidekiq/wiki/Getting-Started) and follow the simple setup process.
33
+ You can watch [this YouTube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
34
+ Sidekiq and see its features in action. Here's the Web UI:
35
+
36
+ ![Web UI](https://github.com/sidekiq/sidekiq/raw/main/examples/web-ui.png)
37
+
38
+ Performance
39
+ ---------------
40
+
41
+ The benchmark in `bin/sidekiqload` creates 500,000 no-op jobs and drains them as fast as possible, assuming a fixed Redis network latency of 1ms.
42
+ This requires a lot of Redis network I/O and JSON parsing.
43
+ This benchmark is IO-bound so we increase the concurrency to 25.
44
+ If your application is sending lots of emails or performing other network-intensive work, you could see a similar benefit but be careful not to saturate the CPU.
49
45
 
50
- ![Web UI](https://github.com/mperham/sidekiq/raw/master/examples/web-ui.png)
46
+ Version | Time to process 500k jobs | Throughput (jobs/sec) | Ruby | Concurrency | Job Type
47
+ -----------------|------|---------|---------|------------------------|---
48
+ Sidekiq 7.0.3 | 21.3 sec| 23,500 | 3.2.0+yjit | 30 | Sidekiq::Job
49
+ Sidekiq 7.0.3 | 33.8 sec| 14,700 | 3.2.0+yjit | 30 | ActiveJob 7.0.4
50
+ Sidekiq 7.0.3 | 23.5 sec| 21,300 | 3.2.0 | 30 | Sidekiq::Job
51
+ Sidekiq 7.0.3 | 46.5 sec| 10,700 | 3.2.0 | 30 | ActiveJob 7.0.4
52
+ Sidekiq 7.0.3 | 23.0 sec| 21,700 | 2.7.5 | 30 | Sidekiq::Job
53
+ Sidekiq 7.0.3 | 46.5 sec| 10,850 | 2.7.5 | 30 | ActiveJob 7.0.4
51
54
 
55
+ Most of Sidekiq's overhead is Redis network I/O.
56
+ ActiveJob adds a notable amount of CPU overhead due to argument deserialization and callbacks.
57
+ Concurrency of 30 was determined experimentally to maximize one CPU without saturating it.
52
58
 
53
59
  Want to Upgrade?
54
60
  -------------------
55
61
 
62
+ Use `bundle up sidekiq` to upgrade Sidekiq and all its dependencies.
63
+ Upgrade notes between each major version can be found in the `docs/` directory.
64
+
56
65
  I also sell Sidekiq Pro and Sidekiq Enterprise, extensions to Sidekiq which provide more
57
66
  features, a commercial-friendly license and allow you to support high
58
67
  quality open source development all at the same time. Please see the
59
- [Sidekiq](http://sidekiq.org/) homepage for more detail.
68
+ [Sidekiq](https://sidekiq.org/) homepage for more detail.
60
69
 
61
70
 
62
- More Information
71
+ Problems?
63
72
  -----------------
64
73
 
65
- Please see the [sidekiq wiki](https://github.com/mperham/sidekiq/wiki) for the official documentation.
66
- [mperham/sidekiq on Gitter](https://gitter.im/mperham/sidekiq) is dedicated to this project,
67
- but bug reports or feature requests suggestions should still go through [issues on Github](https://github.com/mperham/sidekiq/issues). Release announcements are made to the [@sidekiq](https://twitter.com/sidekiq) Twitter account.
74
+ **Please do not directly email any Sidekiq committers with questions or problems.**
75
+ A community is best served when discussions are held in public.
68
76
 
69
- You may also find useful a [Google Group](https://groups.google.com/forum/#!forum/sidekiq) dedicated to Sidekiq discussion and [a Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow.
77
+ If you have a problem, please review the [FAQ](https://github.com/sidekiq/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/sidekiq/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages.
78
+ Searching the [issues](https://github.com/sidekiq/sidekiq/issues) for your problem is also a good idea.
70
79
 
80
+ Sidekiq Pro and Sidekiq Enterprise customers get private email support.
81
+ You can purchase at https://sidekiq.org; email support@contribsys.com for help.
71
82
 
72
- Problems?
73
- -----------------
83
+ Useful resources:
74
84
 
75
- **Please do not directly email any Sidekiq committers with questions or problems.** A community is best served when discussions are held in public.
85
+ * Product documentation is in the [wiki](https://github.com/sidekiq/sidekiq/wiki).
86
+ * Occasional announcements are made to the [@sidekiq](https://ruby.social/@sidekiq) Mastodon account.
87
+ * The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q &amp; A.
76
88
 
77
- If you have a problem, please review the [FAQ](https://github.com/mperham/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/mperham/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages. Searching the issues for your problem is also a good idea. If that doesn't help, feel free to email the Sidekiq mailing list, chat in Gitter, or open a new issue.
78
- The mailing list is the preferred place to ask questions on usage. If you are encountering what you think is a bug, please open an issue.
89
+ Every Friday morning is Sidekiq office hour: I video chat and answer questions.
90
+ See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
79
91
 
80
-
81
- Thanks
92
+ Contributing
82
93
  -----------------
83
94
 
84
- Sidekiq stays fast by using the [JProfiler java profiler](http://www.ej-technologies.com/products/jprofiler/overview.html) to find and fix
85
- performance problems on JRuby. Unfortunately MRI does not have good profile tooling.
86
-
95
+ Please see [the contributing guidelines](https://github.com/sidekiq/sidekiq/blob/main/.github/contributing.md).
87
96
 
88
97
  License
89
98
  -----------------
90
99
 
91
- Please see [LICENSE](https://github.com/mperham/sidekiq/blob/master/LICENSE) for licensing details.
92
-
100
+ Please see [LICENSE.txt](https://github.com/sidekiq/sidekiq/blob/main/LICENSE.txt) for licensing details.
101
+ The license for Sidekiq Pro and Sidekiq Enterprise can be found in [COMM-LICENSE.txt](https://github.com/sidekiq/sidekiq/blob/main/COMM-LICENSE.txt).
93
102
 
94
103
  Author
95
104
  -----------------
96
105
 
97
- Mike Perham, [@mperham](https://twitter.com/mperham) / [@sidekiq](https://twitter.com/sidekiq), [http://www.mikeperham.com](http://www.mikeperham.com) / [http://www.contribsys.com](http://www.contribsys.com)
106
+ Mike Perham, [@getajobmike](https://ruby.social/@getajobmike) / [@sidekiq](https://ruby.social/@sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
data/bin/sidekiq CHANGED
@@ -3,17 +3,35 @@
3
3
  # Quiet some warnings we see when running in warning mode:
4
4
  # RUBYOPT=-w bundle exec sidekiq
5
5
  $TESTING = false
6
- $CELLULOID_DEBUG = false
7
6
 
8
- require_relative '../lib/sidekiq/cli'
7
+ require_relative "../lib/sidekiq/cli"
8
+
9
+ def integrate_with_systemd
10
+ return unless ENV["NOTIFY_SOCKET"]
11
+
12
+ Sidekiq.configure_server do |config|
13
+ config.logger.info "Enabling systemd notification integration"
14
+ require "sidekiq/sd_notify"
15
+ config.on(:startup) do
16
+ Sidekiq::SdNotify.ready
17
+ end
18
+ config.on(:shutdown) do
19
+ Sidekiq::SdNotify.stopping
20
+ end
21
+ Sidekiq.start_watchdog if Sidekiq::SdNotify.watchdog?
22
+ end
23
+ end
9
24
 
10
25
  begin
11
26
  cli = Sidekiq::CLI.instance
12
27
  cli.parse
28
+
29
+ integrate_with_systemd
30
+
13
31
  cli.run
14
32
  rescue => e
15
33
  raise e if $DEBUG
16
- STDERR.puts e.message
17
- STDERR.puts e.backtrace.join("\n")
34
+ warn e.message
35
+ warn e.backtrace.join("\n")
18
36
  exit 1
19
37
  end
data/bin/sidekiqload CHANGED
@@ -1,153 +1,247 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
+ #
4
+ # bin/sidekiqload is a helpful script to load test and
5
+ # performance tune Sidekiq's core. It creates 500,000 no-op
6
+ # jobs and executes them as fast as possible.
7
+ # Example Usage:
8
+ #
9
+ # > RUBY_YJIT_ENABLE=1 LATENCY=0 THREADS=10 bin/sidekiqload
10
+ # Result: Done, 500000 jobs in 20.264945 sec, 24673 jobs/sec
11
+ #
12
+ # Use LATENCY=1 to get a more real world network setup
13
+ # but you'll need to setup and start toxiproxy as noted below.
14
+ #
15
+ # Use AJ=1 to test ActiveJob instead of plain old Sidekiq::Jobs so
16
+ # you can see the runtime performance difference between the two APIs.
17
+ #
18
+ # None of this script is considered a public API and may change over time.
19
+ #
20
+
3
21
  # Quiet some warnings we see when running in warning mode:
4
22
  # RUBYOPT=-w bundle exec sidekiq
5
23
  $TESTING = false
6
- $CELLULOID_DEBUG = false
24
+ puts RUBY_DESCRIPTION
25
+ puts(%w[THREADS LATENCY AJ PROFILE].map { |x| "#{x}: #{ENV[x] || "nil"}" }.join(", "))
26
+
27
+ require "ruby-prof" if ENV["PROFILE"]
28
+ require "bundler/setup"
29
+ Bundler.require(:default, :load_test)
7
30
 
8
- require 'celluloid/current'
9
- puts Celluloid::VERSION
10
- require_relative '../lib/sidekiq/cli'
11
- require_relative '../lib/sidekiq/launcher'
12
- Celluloid.logger = nil
31
+ latency = Integer(ENV["LATENCY"] || 1)
32
+ if latency > 0
33
+ # brew tap shopify/shopify
34
+ # brew install toxiproxy
35
+ # run `toxiproxy-server` in a separate terminal window.
36
+ require "toxiproxy"
37
+ # simulate a non-localhost network for realer-world conditions.
38
+ # adding 1ms of network latency has an ENORMOUS impact on benchmarks
39
+ Toxiproxy.populate([{
40
+ name: "redis",
41
+ listen: "127.0.0.1:6380",
42
+ upstream: "127.0.0.1:6379"
43
+ }])
44
+ end
13
45
 
14
- include Sidekiq::Util
46
+ if ENV["AJ"]
47
+ require "active_job"
48
+ puts "Using ActiveJob #{ActiveJob::VERSION::STRING}"
49
+ ActiveJob::Base.queue_adapter = :sidekiq
50
+ ActiveJob::Base.logger.level = Logger::WARN
15
51
 
16
- Sidekiq.configure_server do |config|
17
- config.redis = { db: 13, port: 6380 }
18
- config.options[:queues] << 'default'
19
- config.logger.level = Logger::ERROR
20
- config.average_scheduled_poll_interval = 2
52
+ class LoadJob < ActiveJob::Base
53
+ def perform(idx, ts = nil)
54
+ puts(Time.now.to_f - ts) if !ts.nil?
55
+ end
56
+ end
21
57
  end
22
58
 
23
59
  class LoadWorker
24
- include Sidekiq::Worker
60
+ include Sidekiq::Job
25
61
  sidekiq_options retry: 1
26
62
  sidekiq_retry_in do |x|
27
63
  1
28
64
  end
29
65
 
30
- def perform(idx)
31
- #raise idx.to_s if idx % 100 == 1
66
+ def perform(idx, ts = nil)
67
+ puts(Time.now.to_f - ts) if !ts.nil?
68
+ # raise idx.to_s if idx % 100 == 1
32
69
  end
33
70
  end
34
71
 
35
- # brew tap shopify/shopify
36
- # brew install toxiproxy
37
- # gem install toxiproxy
38
- require 'toxiproxy'
39
- # simulate a non-localhost network for realer-world conditions.
40
- # adding 1ms of network latency has an ENORMOUS impact on benchmarks
41
- Toxiproxy.populate([{
42
- "name": "redis",
43
- "listen": "127.0.0.1:6380",
44
- "upstream": "127.0.0.1:6379"
45
- }])
46
-
47
- self_read, self_write = IO.pipe
48
- %w(INT TERM USR1 USR2 TTIN).each do |sig|
49
- begin
50
- trap sig do
51
- self_write.puts(sig)
72
+ def Process.rss
73
+ `ps -o rss= -p #{Process.pid}`.chomp.to_i
74
+ end
75
+
76
+ class Loader
77
+ def initialize
78
+ @iter = ENV["GC"] ? 10 : 500
79
+ @count = Integer(ENV["COUNT"] || 1_000)
80
+ @latency = Integer(ENV["LATENCY"] || 1)
81
+ end
82
+
83
+ def configure
84
+ @x = Sidekiq.configure_embed do |config|
85
+ config.redis = {db: 13, port: ((@latency > 0) ? 6380 : 6379)}
86
+ config.concurrency = Integer(ENV.fetch("THREADS", "10"))
87
+ # config.redis = { db: 13, port: 6380, driver: :hiredis}
88
+ config.queues = %w[default]
89
+ config.logger.level = Logger::WARN
90
+ config.average_scheduled_poll_interval = 2
91
+ config.reliable! if defined?(Sidekiq::Pro)
92
+ end
93
+
94
+ @self_read, @self_write = IO.pipe
95
+ %w[INT TERM TSTP TTIN].each do |sig|
96
+ trap sig do
97
+ @self_write.puts(sig)
98
+ end
99
+ rescue ArgumentError
100
+ puts "Signal #{sig} not supported"
52
101
  end
53
- rescue ArgumentError
54
- puts "Signal #{sig} not supported"
55
102
  end
56
- end
57
103
 
58
- Sidekiq.redis {|c| c.flushdb}
59
- def handle_signal(launcher, sig)
60
- Sidekiq.logger.debug "Got #{sig} signal"
61
- case sig
62
- when 'INT'
63
- # Handle Ctrl-C in JRuby like MRI
64
- # http://jira.codehaus.org/browse/JRUBY-4637
65
- raise Interrupt
66
- when 'TERM'
67
- # Heroku sends TERM and then waits 10 seconds for process to exit.
68
- raise Interrupt
69
- when 'USR1'
70
- Sidekiq.logger.info "Received USR1, no longer accepting new work"
71
- launcher.manager.async.stop
72
- #fire_event(:quiet, true)
73
- when 'USR2'
74
- if Sidekiq.options[:logfile]
75
- Sidekiq.logger.info "Received USR2, reopening log file"
76
- Sidekiq::Logging.reopen_logs
104
+ def handle_signal(sig)
105
+ launcher = @x
106
+ Sidekiq.logger.debug "Got #{sig} signal"
107
+ case sig
108
+ when "INT"
109
+ # Handle Ctrl-C in JRuby like MRI
110
+ # http://jira.codehaus.org/browse/JRUBY-4637
111
+ raise Interrupt
112
+ when "TERM"
113
+ # Heroku sends TERM and then waits 30 seconds for process to exit.
114
+ raise Interrupt
115
+ when "TSTP"
116
+ Sidekiq.logger.info "Received TSTP, no longer accepting new work"
117
+ launcher.quiet
118
+ when "TTIN"
119
+ Thread.list.each do |thread|
120
+ Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["label"]}"
121
+ if thread.backtrace
122
+ Sidekiq.logger.warn thread.backtrace.join("\n")
123
+ else
124
+ Sidekiq.logger.warn "<no backtrace available>"
125
+ end
126
+ end
77
127
  end
78
- when 'TTIN'
79
- Thread.list.each do |thread|
80
- Sidekiq.logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
81
- if thread.backtrace
82
- Sidekiq.logger.warn thread.backtrace.join("\n")
83
- else
84
- Sidekiq.logger.warn "<no backtrace available>"
128
+ end
129
+
130
+ def setup
131
+ Sidekiq.logger.error("Setup RSS: #{Process.rss}")
132
+ Sidekiq.redis { |c| c.flushdb }
133
+ start = Time.now
134
+ if ENV["AJ"]
135
+ @iter.times do
136
+ @count.times do |idx|
137
+ LoadJob.perform_later(idx)
138
+ end
139
+ end
140
+ else
141
+ @iter.times do
142
+ arr = Array.new(@count) { |idx| [idx] }
143
+ Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
85
144
  end
86
145
  end
146
+ Sidekiq.logger.warn "Created #{@count * @iter} jobs in #{Time.now - start} sec"
87
147
  end
88
- end
89
148
 
90
- def Process.rss
91
- `ps -o rss= -p #{Process.pid}`.chomp.to_i
92
- end
149
+ def monitor
150
+ @monitor = Thread.new do
151
+ GC.start
152
+ loop do
153
+ sleep 0.2
154
+ qsize = Sidekiq.redis do |conn|
155
+ conn.llen "queue:default"
156
+ end
157
+ total = qsize
158
+ if total == 0
159
+ ending = Time.now - @start
160
+ size = @iter * @count
161
+ Sidekiq.logger.error("Done, #{size} jobs in #{ending} sec, #{(size / ending).to_i} jobs/sec")
162
+ Sidekiq.logger.error("Ending RSS: #{Process.rss}")
163
+ Sidekiq.logger.error("Now here's the latency for three jobs")
93
164
 
94
- iter = 10
95
- count = 10_000
165
+ if ENV["AJ"]
166
+ LoadJob.perform_later(1, Time.now.to_f)
167
+ LoadJob.perform_later(2, Time.now.to_f)
168
+ LoadJob.perform_later(3, Time.now.to_f)
169
+ else
170
+ LoadWorker.perform_async(1, Time.now.to_f)
171
+ LoadWorker.perform_async(2, Time.now.to_f)
172
+ LoadWorker.perform_async(3, Time.now.to_f)
173
+ end
96
174
 
97
- iter.times do
98
- arr = Array.new(count) do
99
- []
175
+ sleep 0.1
176
+ @x.stop
177
+ Process.kill("INT", $$)
178
+ break
179
+ end
180
+ end
181
+ end
100
182
  end
101
- count.times do |idx|
102
- arr[idx][0] = idx
183
+
184
+ def with_latency(latency, &block)
185
+ Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
186
+ if latency > 0
187
+ Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
188
+ else
189
+ yield
190
+ end
103
191
  end
104
- Sidekiq::Client.push_bulk('class' => LoadWorker, 'args' => arr)
105
- end
106
- Sidekiq.logger.error "Created #{count*iter} jobs"
107
-
108
- Monitoring = Thread.new do
109
- watchdog("monitor thread") do
110
- while true
111
- sleep 2
112
- qsize, retries = Sidekiq.redis do |conn|
113
- conn.pipelined do
114
- conn.llen "queue:default"
115
- conn.zcard "retry"
116
- end
117
- end.map(&:to_i)
118
- total = qsize + retries
119
- #GC.start
120
- Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
121
- if total == 0
122
- Sidekiq.logger.error("Done")
123
- exit(0)
192
+
193
+ def run(name)
194
+ Sidekiq.logger.warn("Starting #{name}")
195
+ monitor
196
+
197
+ if ENV["PROFILE"]
198
+ RubyProf.exclude_threads = [@monitor]
199
+ RubyProf.start
200
+ elsif ENV["GC"]
201
+ GC.start
202
+ GC.compact
203
+ GC.disable
204
+ Sidekiq.logger.error("GC Start RSS: #{Process.rss}")
205
+ end
206
+ @start = Time.now
207
+ with_latency(@latency) do
208
+ @x.run
209
+
210
+ while (readable_io = IO.select([@self_read]))
211
+ signal = readable_io.first[0].gets.strip
212
+ handle_signal(signal)
124
213
  end
125
214
  end
215
+ # normal
216
+ rescue Interrupt
217
+ rescue => e
218
+ raise e if $DEBUG
219
+ warn e.message
220
+ warn e.backtrace.join("\n")
221
+ exit 1
222
+ ensure
223
+ @x.stop
126
224
  end
127
- end
128
225
 
129
- begin
130
- #RubyProf::exclude_threads = [ Monitoring ]
131
- #RubyProf.start
132
- Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
133
- Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
134
- launcher = Sidekiq::Launcher.new(Sidekiq.options)
135
- launcher.run
136
-
137
- while readable_io = IO.select([self_read])
138
- signal = readable_io.first[0].gets.strip
139
- handle_signal(launcher, signal)
226
+ def done
227
+ Sidekiq.logger.error("GC End RSS: #{Process.rss}") if ENV["GC"]
228
+ if ENV["PROFILE"]
229
+ Sidekiq.logger.error("Profiling...")
230
+ result = RubyProf.stop
231
+ printer = RubyProf::GraphHtmlPrinter.new(result)
232
+ printer.print(File.new("output.html", "w"), min_percent: 1)
140
233
  end
141
234
  end
142
- rescue SystemExit => e
143
- #Sidekiq.logger.error("Profiling...")
144
- #result = RubyProf.stop
145
- #printer = RubyProf::GraphHtmlPrinter.new(result)
146
- #printer.print(File.new("output.html", "w"), :min_percent => 1)
147
- # normal
148
- rescue => e
149
- raise e if $DEBUG
150
- STDERR.puts e.message
151
- STDERR.puts e.backtrace.join("\n")
152
- exit 1
153
235
  end
236
+
237
+ ll = Loader.new
238
+ ll.configure
239
+
240
+ if ENV["WARM"]
241
+ ll.setup
242
+ ll.run("warmup")
243
+ end
244
+
245
+ ll.setup
246
+ ll.run("load")
247
+ ll.done
data/bin/sidekiqmon ADDED
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "sidekiq/monitor"
4
+
5
+ # disable the Redis connection pool logging
6
+ Sidekiq.default_configuration.logger.level = :warn
7
+
8
+ section = "all"
9
+ section = ARGV[0] if ARGV.size == 1
10
+
11
+ Sidekiq::Monitor::Status.new.display(section)
@@ -0,0 +1,57 @@
1
+ require "rails/generators/named_base"
2
+
3
+ module Sidekiq
4
+ module Generators # :nodoc:
5
+ class JobGenerator < ::Rails::Generators::NamedBase # :nodoc:
6
+ desc "This generator creates a Sidekiq Job in app/sidekiq and a corresponding test"
7
+
8
+ check_class_collision suffix: "Job"
9
+
10
+ def self.default_generator_root
11
+ File.dirname(__FILE__)
12
+ end
13
+
14
+ def create_job_file
15
+ template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
16
+ end
17
+
18
+ def create_test_file
19
+ return unless test_framework
20
+
21
+ if test_framework == :rspec
22
+ create_job_spec
23
+ else
24
+ create_job_test
25
+ end
26
+ end
27
+
28
+ private
29
+
30
+ def create_job_spec
31
+ template_file = File.join(
32
+ "spec/sidekiq",
33
+ class_path,
34
+ "#{file_name}_job_spec.rb"
35
+ )
36
+ template "job_spec.rb.erb", template_file
37
+ end
38
+
39
+ def create_job_test
40
+ template_file = File.join(
41
+ "test/sidekiq",
42
+ class_path,
43
+ "#{file_name}_job_test.rb"
44
+ )
45
+ template "job_test.rb.erb", template_file
46
+ end
47
+
48
+ def file_name
49
+ @_file_name ||= super.sub(/_?job\z/i, "")
50
+ end
51
+
52
+ def test_framework
53
+ ::Rails.application.config.generators.options[:rails][:test_framework]
54
+ end
55
+ end
56
+ end
57
+ end
@@ -1,6 +1,6 @@
1
1
  <% module_namespacing do -%>
2
- class <%= class_name %>Worker
3
- include Sidekiq::Worker
2
+ class <%= class_name %>Job
3
+ include Sidekiq::Job
4
4
 
5
5
  def perform(*args)
6
6
  # Do something
@@ -0,0 +1,6 @@
1
+ require 'rails_helper'
2
+ <% module_namespacing do -%>
3
+ RSpec.describe <%= class_name %>Job, type: :job do
4
+ pending "add some examples to (or delete) #{__FILE__}"
5
+ end
6
+ <% end -%>
@@ -0,0 +1,8 @@
1
+ require 'test_helper'
2
+ <% module_namespacing do -%>
3
+ class <%= class_name %>JobTest < Minitest::Test
4
+ def test_example
5
+ skip "add some examples to (or delete) #{__FILE__}"
6
+ end
7
+ end
8
+ <% end -%>