sidekiq 5.2.8 → 6.1.3

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/.github/ISSUE_TEMPLATE/bug_report.md +20 -0
  3. data/.github/workflows/ci.yml +41 -0
  4. data/.gitignore +0 -2
  5. data/.standard.yml +20 -0
  6. data/5.0-Upgrade.md +1 -1
  7. data/6.0-Upgrade.md +72 -0
  8. data/Changes.md +196 -0
  9. data/Ent-2.0-Upgrade.md +37 -0
  10. data/Ent-Changes.md +72 -1
  11. data/Gemfile +12 -11
  12. data/Gemfile.lock +193 -0
  13. data/Pro-5.0-Upgrade.md +25 -0
  14. data/Pro-Changes.md +56 -2
  15. data/README.md +18 -34
  16. data/Rakefile +5 -4
  17. data/bin/sidekiq +26 -2
  18. data/bin/sidekiqload +32 -24
  19. data/bin/sidekiqmon +8 -0
  20. data/lib/generators/sidekiq/templates/worker_test.rb.erb +1 -1
  21. data/lib/generators/sidekiq/worker_generator.rb +21 -13
  22. data/lib/sidekiq/api.rb +245 -219
  23. data/lib/sidekiq/cli.rb +144 -180
  24. data/lib/sidekiq/client.rb +68 -48
  25. data/lib/sidekiq/delay.rb +5 -6
  26. data/lib/sidekiq/exception_handler.rb +10 -12
  27. data/lib/sidekiq/extensions/action_mailer.rb +13 -22
  28. data/lib/sidekiq/extensions/active_record.rb +13 -10
  29. data/lib/sidekiq/extensions/class_methods.rb +14 -11
  30. data/lib/sidekiq/extensions/generic_proxy.rb +4 -4
  31. data/lib/sidekiq/fetch.rb +29 -30
  32. data/lib/sidekiq/job_logger.rb +45 -7
  33. data/lib/sidekiq/job_retry.rb +62 -61
  34. data/lib/sidekiq/launcher.rb +112 -54
  35. data/lib/sidekiq/logger.rb +166 -0
  36. data/lib/sidekiq/manager.rb +11 -13
  37. data/lib/sidekiq/middleware/chain.rb +15 -5
  38. data/lib/sidekiq/middleware/i18n.rb +5 -7
  39. data/lib/sidekiq/monitor.rb +133 -0
  40. data/lib/sidekiq/paginator.rb +18 -14
  41. data/lib/sidekiq/processor.rb +71 -70
  42. data/lib/sidekiq/rails.rb +29 -37
  43. data/lib/sidekiq/redis_connection.rb +50 -48
  44. data/lib/sidekiq/scheduled.rb +28 -29
  45. data/lib/sidekiq/sd_notify.rb +149 -0
  46. data/lib/sidekiq/systemd.rb +24 -0
  47. data/lib/sidekiq/testing/inline.rb +2 -1
  48. data/lib/sidekiq/testing.rb +35 -24
  49. data/lib/sidekiq/util.rb +17 -16
  50. data/lib/sidekiq/version.rb +2 -1
  51. data/lib/sidekiq/web/action.rb +14 -10
  52. data/lib/sidekiq/web/application.rb +74 -72
  53. data/lib/sidekiq/web/csrf_protection.rb +156 -0
  54. data/lib/sidekiq/web/helpers.rb +97 -77
  55. data/lib/sidekiq/web/router.rb +18 -17
  56. data/lib/sidekiq/web.rb +53 -53
  57. data/lib/sidekiq/worker.rb +126 -102
  58. data/lib/sidekiq.rb +69 -44
  59. data/sidekiq.gemspec +15 -16
  60. data/web/assets/javascripts/application.js +25 -27
  61. data/web/assets/javascripts/dashboard.js +4 -23
  62. data/web/assets/stylesheets/application-dark.css +149 -0
  63. data/web/assets/stylesheets/application.css +28 -6
  64. data/web/locales/de.yml +14 -2
  65. data/web/locales/en.yml +2 -0
  66. data/web/locales/fr.yml +3 -3
  67. data/web/locales/ja.yml +4 -1
  68. data/web/locales/lt.yml +83 -0
  69. data/web/locales/pl.yml +4 -4
  70. data/web/locales/ru.yml +4 -0
  71. data/web/locales/vi.yml +83 -0
  72. data/web/views/_job_info.erb +2 -1
  73. data/web/views/busy.erb +8 -3
  74. data/web/views/dead.erb +2 -2
  75. data/web/views/layout.erb +1 -0
  76. data/web/views/morgue.erb +5 -2
  77. data/web/views/queue.erb +10 -1
  78. data/web/views/queues.erb +9 -1
  79. data/web/views/retries.erb +5 -2
  80. data/web/views/retry.erb +2 -2
  81. data/web/views/scheduled.erb +5 -2
  82. metadata +31 -49
  83. data/.circleci/config.yml +0 -61
  84. data/.github/issue_template.md +0 -11
  85. data/.travis.yml +0 -11
  86. data/bin/sidekiqctl +0 -20
  87. data/lib/sidekiq/core_ext.rb +0 -1
  88. data/lib/sidekiq/ctl.rb +0 -221
  89. data/lib/sidekiq/logging.rb +0 -122
  90. data/lib/sidekiq/middleware/server/active_record.rb +0 -23
data/Gemfile.lock ADDED
@@ -0,0 +1,193 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ sidekiq (6.1.2)
5
+ connection_pool (>= 2.2.2)
6
+ rack (~> 2.0)
7
+ redis (>= 4.2.0)
8
+
9
+ GEM
10
+ remote: https://rubygems.org/
11
+ specs:
12
+ actioncable (6.0.3.2)
13
+ actionpack (= 6.0.3.2)
14
+ nio4r (~> 2.0)
15
+ websocket-driver (>= 0.6.1)
16
+ actionmailbox (6.0.3.2)
17
+ actionpack (= 6.0.3.2)
18
+ activejob (= 6.0.3.2)
19
+ activerecord (= 6.0.3.2)
20
+ activestorage (= 6.0.3.2)
21
+ activesupport (= 6.0.3.2)
22
+ mail (>= 2.7.1)
23
+ actionmailer (6.0.3.2)
24
+ actionpack (= 6.0.3.2)
25
+ actionview (= 6.0.3.2)
26
+ activejob (= 6.0.3.2)
27
+ mail (~> 2.5, >= 2.5.4)
28
+ rails-dom-testing (~> 2.0)
29
+ actionpack (6.0.3.2)
30
+ actionview (= 6.0.3.2)
31
+ activesupport (= 6.0.3.2)
32
+ rack (~> 2.0, >= 2.0.8)
33
+ rack-test (>= 0.6.3)
34
+ rails-dom-testing (~> 2.0)
35
+ rails-html-sanitizer (~> 1.0, >= 1.2.0)
36
+ actiontext (6.0.3.2)
37
+ actionpack (= 6.0.3.2)
38
+ activerecord (= 6.0.3.2)
39
+ activestorage (= 6.0.3.2)
40
+ activesupport (= 6.0.3.2)
41
+ nokogiri (>= 1.8.5)
42
+ actionview (6.0.3.2)
43
+ activesupport (= 6.0.3.2)
44
+ builder (~> 3.1)
45
+ erubi (~> 1.4)
46
+ rails-dom-testing (~> 2.0)
47
+ rails-html-sanitizer (~> 1.1, >= 1.2.0)
48
+ activejob (6.0.3.2)
49
+ activesupport (= 6.0.3.2)
50
+ globalid (>= 0.3.6)
51
+ activemodel (6.0.3.2)
52
+ activesupport (= 6.0.3.2)
53
+ activerecord (6.0.3.2)
54
+ activemodel (= 6.0.3.2)
55
+ activesupport (= 6.0.3.2)
56
+ activestorage (6.0.3.2)
57
+ actionpack (= 6.0.3.2)
58
+ activejob (= 6.0.3.2)
59
+ activerecord (= 6.0.3.2)
60
+ marcel (~> 0.3.1)
61
+ activesupport (6.0.3.2)
62
+ concurrent-ruby (~> 1.0, >= 1.0.2)
63
+ i18n (>= 0.7, < 2)
64
+ minitest (~> 5.1)
65
+ tzinfo (~> 1.1)
66
+ zeitwerk (~> 2.2, >= 2.2.2)
67
+ ast (2.4.1)
68
+ builder (3.2.4)
69
+ codecov (0.2.8)
70
+ json
71
+ simplecov
72
+ concurrent-ruby (1.1.7)
73
+ connection_pool (2.2.3)
74
+ crass (1.0.6)
75
+ docile (1.3.2)
76
+ erubi (1.9.0)
77
+ globalid (0.4.2)
78
+ activesupport (>= 4.2.0)
79
+ hiredis (0.6.3)
80
+ i18n (1.8.5)
81
+ concurrent-ruby (~> 1.0)
82
+ json (2.3.1)
83
+ loofah (2.7.0)
84
+ crass (~> 1.0.2)
85
+ nokogiri (>= 1.5.9)
86
+ mail (2.7.1)
87
+ mini_mime (>= 0.1.1)
88
+ marcel (0.3.3)
89
+ mimemagic (~> 0.3.2)
90
+ method_source (1.0.0)
91
+ mimemagic (0.3.5)
92
+ mini_mime (1.0.2)
93
+ mini_portile2 (2.4.0)
94
+ minitest (5.14.3)
95
+ nio4r (2.5.2)
96
+ nokogiri (1.10.10)
97
+ mini_portile2 (~> 2.4.0)
98
+ parallel (1.20.1)
99
+ parser (2.7.2.0)
100
+ ast (~> 2.4.1)
101
+ rack (2.2.3)
102
+ rack-test (1.1.0)
103
+ rack (>= 1.0, < 3)
104
+ rails (6.0.3.2)
105
+ actioncable (= 6.0.3.2)
106
+ actionmailbox (= 6.0.3.2)
107
+ actionmailer (= 6.0.3.2)
108
+ actionpack (= 6.0.3.2)
109
+ actiontext (= 6.0.3.2)
110
+ actionview (= 6.0.3.2)
111
+ activejob (= 6.0.3.2)
112
+ activemodel (= 6.0.3.2)
113
+ activerecord (= 6.0.3.2)
114
+ activestorage (= 6.0.3.2)
115
+ activesupport (= 6.0.3.2)
116
+ bundler (>= 1.3.0)
117
+ railties (= 6.0.3.2)
118
+ sprockets-rails (>= 2.0.0)
119
+ rails-dom-testing (2.0.3)
120
+ activesupport (>= 4.2.0)
121
+ nokogiri (>= 1.6)
122
+ rails-html-sanitizer (1.3.0)
123
+ loofah (~> 2.3)
124
+ railties (6.0.3.2)
125
+ actionpack (= 6.0.3.2)
126
+ activesupport (= 6.0.3.2)
127
+ method_source
128
+ rake (>= 0.8.7)
129
+ thor (>= 0.20.3, < 2.0)
130
+ rainbow (3.0.0)
131
+ rake (13.0.1)
132
+ redis (4.2.5)
133
+ redis-namespace (1.6.0)
134
+ redis (>= 3.0.4)
135
+ regexp_parser (2.0.0)
136
+ rexml (3.2.4)
137
+ rubocop (1.4.2)
138
+ parallel (~> 1.10)
139
+ parser (>= 2.7.1.5)
140
+ rainbow (>= 2.2.2, < 4.0)
141
+ regexp_parser (>= 1.8)
142
+ rexml
143
+ rubocop-ast (>= 1.1.1)
144
+ ruby-progressbar (~> 1.7)
145
+ unicode-display_width (>= 1.4.0, < 2.0)
146
+ rubocop-ast (1.3.0)
147
+ parser (>= 2.7.1.5)
148
+ rubocop-performance (1.9.1)
149
+ rubocop (>= 0.90.0, < 2.0)
150
+ rubocop-ast (>= 0.4.0)
151
+ ruby-progressbar (1.10.1)
152
+ simplecov (0.19.0)
153
+ docile (~> 1.1)
154
+ simplecov-html (~> 0.11)
155
+ simplecov-html (0.12.3)
156
+ sprockets (4.0.2)
157
+ concurrent-ruby (~> 1.0)
158
+ rack (> 1, < 3)
159
+ sprockets-rails (3.2.1)
160
+ actionpack (>= 4.0)
161
+ activesupport (>= 4.0)
162
+ sprockets (>= 3.0.0)
163
+ sqlite3 (1.4.2)
164
+ standard (0.10.2)
165
+ rubocop (= 1.4.2)
166
+ rubocop-performance (= 1.9.1)
167
+ thor (1.0.1)
168
+ thread_safe (0.3.6)
169
+ toxiproxy (1.0.3)
170
+ tzinfo (1.2.7)
171
+ thread_safe (~> 0.1)
172
+ unicode-display_width (1.7.0)
173
+ websocket-driver (0.7.3)
174
+ websocket-extensions (>= 0.1.0)
175
+ websocket-extensions (0.1.5)
176
+ zeitwerk (2.4.0)
177
+
178
+ PLATFORMS
179
+ ruby
180
+
181
+ DEPENDENCIES
182
+ activerecord-jdbcsqlite3-adapter
183
+ codecov
184
+ hiredis
185
+ minitest
186
+ rails (>= 6.0.2)
187
+ rake
188
+ redis-namespace
189
+ sidekiq!
190
+ simplecov
191
+ sqlite3
192
+ standard
193
+ toxiproxy
@@ -0,0 +1,25 @@
1
+ # Welcome to Sidekiq Pro 5.0!
2
+
3
+ Sidekiq Pro 5.0 is mainly a cleanup release for Sidekiq 6.0. The
4
+ migration should be as close to trivial as a major version bump can be.
5
+ Note that Sidekiq 6.0 does have major breaking changes.
6
+
7
+ ## What's New
8
+
9
+ * New localizations for the Sidekiq Pro Web UI: ES, ZH, PT, JA, RU
10
+ * Removed deprecated APIs and warnings.
11
+ * Various changes for Sidekiq 6.0
12
+ * Requires Ruby 2.5+ and Redis 4.0+
13
+ * Requires Sidekiq 6.0+.
14
+
15
+ ## Upgrade
16
+
17
+ * Upgrade to the latest Sidekiq Pro 4.x.
18
+ ```ruby
19
+ gem 'sidekiq-pro', '< 5'
20
+ ```
21
+ * Fix any deprecation warnings you see.
22
+ * Upgrade to 5.x.
23
+ ```ruby
24
+ gem 'sidekiq-pro', '< 6'
25
+ ```
data/Pro-Changes.md CHANGED
@@ -2,11 +2,65 @@
2
2
 
3
3
  [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
4
4
 
5
- Please see [http://sidekiq.org/](http://sidekiq.org/) for more details and how to buy.
5
+ Please see [sidekiq.org](https://sidekiq.org/) for more details and how to buy.
6
6
 
7
- HEAD
7
+ 5.2.1
8
8
  ---------
9
9
 
10
+ - Propagate death callbacks to parent batches [#4774]
11
+ - Allow customization of Batch linger to quickly reclaim memory in Redis [#4772]
12
+ - Fix disappearing processes in Busy due to super_fetch initialization when used in
13
+ tandem with `SIDEKIQ_PRELOAD_APP=1` in `sidekiqswarm`. [#4733]
14
+
15
+ 5.2.0
16
+ ---------
17
+
18
+ - The Sidekiq Pro and Enterprise gem servers now `bundle install` much faster with **Bundler 2.2+** [#4158]
19
+ - Fix issue with reliable push and multiple shards [#4669]
20
+ - Fix Pro memory leak due to fetch refactoring in Sidekiq 6.1 [#4652]
21
+ - Gracefully handle poison pill jobs [#4633]
22
+ - Remove support for multi-shard batches [#4642]
23
+ - Rename `Sidekiq::Rack::BatchStatus` to `Sidekiq::Pro::BatchStatus` [#4655]
24
+
25
+ 5.1.1
26
+ ---------
27
+
28
+ - Fix broken basic fetcher [#4616]
29
+
30
+ 5.1.0
31
+ ---------
32
+
33
+ - Remove old Statsd metrics with `WorkerName` in the name [#4377]
34
+ ```
35
+ job.WorkerName.count -> job.count with tag worker:WorkerName
36
+ job.WorkerName.perform -> job.perform with tag worker:WorkerName
37
+ job.WorkerName.failure -> job.failure with tag worker:WorkerName
38
+ ```
39
+ - Remove `concurrent-ruby` gem dependency [#4586]
40
+ - Update `constantize` for batch callbacks. [#4469]
41
+ - Add queue tag to `jobs.recovered.fetch` metric [#4594]
42
+ - Refactor Pro's fetch infrastructure [#4602]
43
+
44
+ 5.0.1
45
+ ---------
46
+
47
+ - Rejigger batch failures UI to add direct links to retries and scheduled jobs [#4209]
48
+ - Delete batch data with `UNLINK` [#4155]
49
+ - Fix bug where a scheduled job can lose its scheduled time when using reliable push [#4267]
50
+ - Sidekiq::JobSet#scan and #find_job APIs have been promoted to Sidekiq OSS. [#4259]
51
+
52
+ 5.0.0
53
+ ---------
54
+
55
+ - There is no significant migration from Sidekiq Pro 4.0 to 5.0
56
+ but make sure you read the [update notes for Sidekiq
57
+ 6.0](https://github.com/mperham/sidekiq/blob/master/6.0-Upgrade.md).
58
+ - Removed various deprecated APIs and associated warnings.
59
+ - **BREAKING CHANGE** Remove the `Sidekiq::Batch::Status#dead_jobs` API in favor of
60
+ `Sidekiq::Batch::Status#dead_jids`. [#4217]
61
+ - Update Sidekiq Pro codebase to use StandardRB formatting
62
+ - Fix lingering "b-XXX-died" elements in Redis which could cause
63
+ excessive memory usage. [#4217]
10
64
  - Add ES translations, see issues [#3949](https://github.com/mperham/sidekiq/issues/3949) and [#3951](https://github.com/mperham/sidekiq/issues/3951) to add your own language.
11
65
 
12
66
  4.0.5
data/README.md CHANGED
@@ -2,10 +2,7 @@ Sidekiq
2
2
  ==============
3
3
 
4
4
  [![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
5
- [![Code Climate](https://codeclimate.com/github/mperham/sidekiq.svg)](https://codeclimate.com/github/mperham/sidekiq)
6
- [![Build Status](https://circleci.com/gh/mperham/sidekiq/tree/master.svg?style=svg)](https://circleci.com/gh/mperham/sidekiq/tree/master)
7
- [![Gitter Chat](https://badges.gitter.im/mperham/sidekiq.svg)](https://gitter.im/mperham/sidekiq)
8
-
5
+ ![Build](https://github.com/mperham/sidekiq/workflows/CI/badge.svg)
9
6
 
10
7
  Simple, efficient background processing for Ruby.
11
8
 
@@ -13,32 +10,27 @@ Sidekiq uses threads to handle many jobs at the same time in the
13
10
  same process. It does not require Rails but will integrate tightly with
14
11
  Rails to make background processing dead simple.
15
12
 
16
- Sidekiq is compatible with Resque. It uses the exact same
17
- message format as Resque so it can integrate into an existing Resque processing farm.
18
- You can have Sidekiq and Resque run side-by-side at the same time and
19
- use the Resque client to enqueue jobs in Redis to be processed by Sidekiq.
20
-
21
13
  Performance
22
14
  ---------------
23
15
 
24
- Version | Latency | Garbage created for 10,000 jobs | Time to process 100,000 jobs | Throughput
25
- -----------------|------|---------|---------|------------------------
26
- Sidekiq 4.0.0 | 10ms | 151 MB | 22 sec | **4500 jobs/sec**
27
- Sidekiq 3.5.1 | 22ms | 1257 MB | 125 sec | 800 jobs/sec
28
- Resque 1.25.2 | - | - | 420 sec | 240 jobs/sec
29
- DelayedJob 4.1.1 | - | - | 465 sec | 215 jobs/sec
16
+ Version | Latency | Garbage created for 10k jobs | Time to process 100k jobs | Throughput | Ruby
17
+ -----------------|------|---------|---------|------------------------|-----
18
+ Sidekiq 6.0.2 | 3 ms | 156 MB | 14.0 sec| **7100 jobs/sec** | MRI 2.6.3
19
+ Sidekiq 6.0.0 | 3 ms | 156 MB | 19 sec | 5200 jobs/sec | MRI 2.6.3
20
+ Sidekiq 4.0.0 | 10 ms | 151 MB | 22 sec | 4500 jobs/sec |
21
+ Sidekiq 3.5.1 | 22 ms | 1257 MB | 125 sec | 800 jobs/sec |
22
+ Resque 1.25.2 | - | - | 420 sec | 240 jobs/sec |
23
+ DelayedJob 4.1.1 | - | - | 465 sec | 215 jobs/sec |
30
24
 
31
- <small>This benchmark can be found in `bin/sidekiqload`.</small>
25
+ This benchmark can be found in `bin/sidekiqload` and assumes a Redis network latency of 1ms.
32
26
 
33
27
  Requirements
34
28
  -----------------
35
29
 
36
- Sidekiq supports CRuby 2.2.2+ and JRuby 9k.
30
+ - Redis: 4.0+
31
+ - Ruby: MRI 2.5+ or JRuby 9.2+.
37
32
 
38
- All Rails releases >= 4.0 are officially supported.
39
-
40
- Redis 2.8 or greater is required. 3.0.3+ is recommended for large
41
- installations with thousands of worker threads.
33
+ Sidekiq 6.0 supports Rails 5.0+ but does not require it.
42
34
 
43
35
 
44
36
  Installation
@@ -63,7 +55,7 @@ Want to Upgrade?
63
55
  I also sell Sidekiq Pro and Sidekiq Enterprise, extensions to Sidekiq which provide more
64
56
  features, a commercial-friendly license and allow you to support high
65
57
  quality open source development all at the same time. Please see the
66
- [Sidekiq](http://sidekiq.org/) homepage for more detail.
58
+ [Sidekiq](https://sidekiq.org/) homepage for more detail.
67
59
 
68
60
  Subscribe to the **[quarterly newsletter](https://tinyletter.com/sidekiq)** to stay informed about the latest
69
61
  features and changes to Sidekiq and its bigger siblings.
@@ -77,24 +69,16 @@ Problems?
77
69
  If you have a problem, please review the [FAQ](https://github.com/mperham/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/mperham/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages.
78
70
  Searching the [issues](https://github.com/mperham/sidekiq/issues) for your problem is also a good idea.
79
71
 
80
- Sidekiq Pro and Sidekiq Enterprise customers get private email support. You can purchase at http://sidekiq.org; email support@contribsys.com for help.
72
+ Sidekiq Pro and Sidekiq Enterprise customers get private email support. You can purchase at https://sidekiq.org; email support@contribsys.com for help.
81
73
 
82
74
  Useful resources:
83
75
 
84
76
  * Product documentation is in the [wiki](https://github.com/mperham/sidekiq/wiki).
85
- * Release announcements are made to the [@sidekiq](https://twitter.com/sidekiq) Twitter account.
77
+ * Occasional announcements are made to the [@sidekiq](https://twitter.com/sidekiq) Twitter account.
86
78
  * The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q &amp; A.
87
79
 
88
- **No support via Twitter**
89
-
90
80
  Every Friday morning is Sidekiq happy hour: I video chat and answer questions.
91
- See the [Sidekiq support page](http://sidekiq.org/support.html) for details.
92
-
93
- Thanks
94
- -----------------
95
-
96
- Sidekiq stays fast by using the [JProfiler java profiler](http://www.ej-technologies.com/products/jprofiler/overview.html) to find and fix
97
- performance problems on JRuby. Unfortunately MRI does not have good multithreaded profiling tools.
81
+ See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
98
82
 
99
83
 
100
84
  License
@@ -106,4 +90,4 @@ Please see [LICENSE](https://github.com/mperham/sidekiq/blob/master/LICENSE) for
106
90
  Author
107
91
  -----------------
108
92
 
109
- Mike Perham, [@mperham](https://twitter.com/mperham) / [@sidekiq](https://twitter.com/sidekiq), [http://www.mikeperham.com](http://www.mikeperham.com) / [http://www.contribsys.com](http://www.contribsys.com)
93
+ Mike Perham, [@getajobmike](https://twitter.com/getajobmike) / [@sidekiq](https://twitter.com/sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
data/Rakefile CHANGED
@@ -1,9 +1,10 @@
1
- require 'bundler/gem_tasks'
2
- require 'rake/testtask'
1
+ require "bundler/gem_tasks"
2
+ require "rake/testtask"
3
+ require "standard/rake"
3
4
 
4
5
  Rake::TestTask.new(:test) do |test|
5
6
  test.warning = true
6
- test.pattern = 'test/**/test_*.rb'
7
+ test.pattern = "test/**/test_*.rb"
7
8
  end
8
9
 
9
- task default: :test
10
+ task default: [:standard, :test]
data/bin/sidekiq CHANGED
@@ -6,13 +6,37 @@ $TESTING = false
6
6
 
7
7
  require_relative '../lib/sidekiq/cli'
8
8
 
9
+ def integrate_with_systemd
10
+ return unless ENV["NOTIFY_SOCKET"]
11
+
12
+ Sidekiq.configure_server do |config|
13
+ Sidekiq.logger.info "Enabling systemd notification integration"
14
+ require "sidekiq/sd_notify"
15
+ config.on(:startup) do
16
+ Sidekiq::SdNotify.ready
17
+ end
18
+ config.on(:shutdown) do
19
+ Sidekiq::SdNotify.stopping
20
+ end
21
+ Sidekiq.start_watchdog if Sidekiq::SdNotify.watchdog?
22
+ end
23
+ end
24
+
9
25
  begin
10
26
  cli = Sidekiq::CLI.instance
11
27
  cli.parse
28
+
29
+ integrate_with_systemd
30
+
12
31
  cli.run
13
32
  rescue => e
14
33
  raise e if $DEBUG
15
- STDERR.puts e.message
16
- STDERR.puts e.backtrace.join("\n")
34
+ if Sidekiq.error_handlers.length == 0
35
+ STDERR.puts e.message
36
+ STDERR.puts e.backtrace.join("\n")
37
+ else
38
+ cli.handle_exception e
39
+ end
40
+
17
41
  exit 1
18
42
  end
data/bin/sidekiqload CHANGED
@@ -5,7 +5,8 @@
5
5
  $TESTING = false
6
6
 
7
7
  #require 'ruby-prof'
8
- Bundler.require(:default)
8
+ require 'bundler/setup'
9
+ Bundler.require(:default, :load_test)
9
10
 
10
11
  require_relative '../lib/sidekiq/cli'
11
12
  require_relative '../lib/sidekiq/launcher'
@@ -13,8 +14,8 @@ require_relative '../lib/sidekiq/launcher'
13
14
  include Sidekiq::Util
14
15
 
15
16
  Sidekiq.configure_server do |config|
16
- #config.options[:concurrency] = 1
17
- config.redis = { db: 13 }
17
+ config.options[:concurrency] = 10
18
+ config.redis = { db: 13, port: 6380, driver: :hiredis }
18
19
  config.options[:queues] << 'default'
19
20
  config.logger.level = Logger::ERROR
20
21
  config.average_scheduled_poll_interval = 2
@@ -28,7 +29,8 @@ class LoadWorker
28
29
  1
29
30
  end
30
31
 
31
- def perform(idx)
32
+ def perform(idx, ts=nil)
33
+ puts(Time.now.to_f - ts) if ts != nil
32
34
  #raise idx.to_s if idx % 100 == 1
33
35
  end
34
36
  end
@@ -36,14 +38,15 @@ end
36
38
  # brew tap shopify/shopify
37
39
  # brew install toxiproxy
38
40
  # gem install toxiproxy
39
- #require 'toxiproxy'
41
+ # run `toxiproxy-server` in a separate terminal window.
42
+ require 'toxiproxy'
40
43
  # simulate a non-localhost network for realer-world conditions.
41
44
  # adding 1ms of network latency has an ENORMOUS impact on benchmarks
42
- #Toxiproxy.populate([{
43
- #"name": "redis",
44
- #"listen": "127.0.0.1:6380",
45
- #"upstream": "127.0.0.1:6379"
46
- #}])
45
+ Toxiproxy.populate([{
46
+ "name": "redis",
47
+ "listen": "127.0.0.1:6380",
48
+ "upstream": "127.0.0.1:6379"
49
+ }])
47
50
 
48
51
  self_read, self_write = IO.pipe
49
52
  %w(INT TERM TSTP TTIN).each do |sig|
@@ -100,21 +103,26 @@ iter.times do
100
103
  end
101
104
  Sidekiq.logger.error "Created #{count*iter} jobs"
102
105
 
106
+ start = Time.now
107
+
103
108
  Monitoring = Thread.new do
104
109
  watchdog("monitor thread") do
105
110
  while true
106
- sleep 1
107
- qsize, retries = Sidekiq.redis do |conn|
108
- conn.pipelined do
109
- conn.llen "queue:default"
110
- conn.zcard "retry"
111
- end
112
- end.map(&:to_i)
113
- total = qsize + retries
114
- #GC.start
115
- Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
111
+ sleep 0.2
112
+ qsize = Sidekiq.redis do |conn|
113
+ conn.llen "queue:default"
114
+ end
115
+ total = qsize
116
+ #Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
116
117
  if total == 0
117
- Sidekiq.logger.error("Done")
118
+ Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
119
+ Sidekiq.logger.error("Now here's the latency for three jobs")
120
+
121
+ LoadWorker.perform_async(1, Time.now.to_f)
122
+ LoadWorker.perform_async(2, Time.now.to_f)
123
+ LoadWorker.perform_async(3, Time.now.to_f)
124
+
125
+ sleep 0.2
118
126
  exit(0)
119
127
  end
120
128
  end
@@ -125,8 +133,8 @@ begin
125
133
  #RubyProf::exclude_threads = [ Monitoring ]
126
134
  #RubyProf.start
127
135
  fire_event(:startup)
128
- #Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
129
- #Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
136
+ Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
137
+ Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
130
138
  launcher = Sidekiq::Launcher.new(Sidekiq.options)
131
139
  launcher.run
132
140
 
@@ -134,7 +142,7 @@ begin
134
142
  signal = readable_io.first[0].gets.strip
135
143
  handle_signal(launcher, signal)
136
144
  end
137
- #end
145
+ end
138
146
  rescue SystemExit => e
139
147
  #Sidekiq.logger.error("Profiling...")
140
148
  #result = RubyProf.stop
data/bin/sidekiqmon ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'sidekiq/monitor'
4
+
5
+ section = "all"
6
+ section = ARGV[0] if ARGV.size == 1
7
+
8
+ Sidekiq::Monitor::Status.new.display(section)
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
  <% module_namespacing do -%>
3
- class <%= class_name %>WorkerTest < <% if defined? Minitest::Test %>Minitest::Test<% else %>MiniTest::Unit::TestCase<% end %>
3
+ class <%= class_name %>WorkerTest < Minitest::Test
4
4
  def test_example
5
5
  skip "add some examples to (or delete) #{__FILE__}"
6
6
  end
@@ -1,22 +1,24 @@
1
- require 'rails/generators/named_base'
1
+ require "rails/generators/named_base"
2
2
 
3
3
  module Sidekiq
4
4
  module Generators # :nodoc:
5
5
  class WorkerGenerator < ::Rails::Generators::NamedBase # :nodoc:
6
- desc 'This generator creates a Sidekiq Worker in app/workers and a corresponding test'
6
+ desc "This generator creates a Sidekiq Worker in app/workers and a corresponding test"
7
7
 
8
- check_class_collision suffix: 'Worker'
8
+ check_class_collision suffix: "Worker"
9
9
 
10
10
  def self.default_generator_root
11
11
  File.dirname(__FILE__)
12
12
  end
13
13
 
14
14
  def create_worker_file
15
- template 'worker.rb.erb', File.join('app/workers', class_path, "#{file_name}_worker.rb")
15
+ template "worker.rb.erb", File.join("app/workers", class_path, "#{file_name}_worker.rb")
16
16
  end
17
17
 
18
18
  def create_test_file
19
- if defined?(RSpec)
19
+ return unless test_framework
20
+
21
+ if test_framework == :rspec
20
22
  create_worker_spec
21
23
  else
22
24
  create_worker_test
@@ -27,23 +29,29 @@ module Sidekiq
27
29
 
28
30
  def create_worker_spec
29
31
  template_file = File.join(
30
- 'spec/workers',
31
- class_path,
32
- "#{file_name}_worker_spec.rb"
32
+ "spec/workers",
33
+ class_path,
34
+ "#{file_name}_worker_spec.rb"
33
35
  )
34
- template 'worker_spec.rb.erb', template_file
36
+ template "worker_spec.rb.erb", template_file
35
37
  end
36
38
 
37
39
  def create_worker_test
38
40
  template_file = File.join(
39
- 'test/workers',
40
- class_path,
41
- "#{file_name}_worker_test.rb"
41
+ "test/workers",
42
+ class_path,
43
+ "#{file_name}_worker_test.rb"
42
44
  )
43
- template 'worker_test.rb.erb', template_file
45
+ template "worker_test.rb.erb", template_file
44
46
  end
45
47
 
48
+ def file_name
49
+ @_file_name ||= super.sub(/_?worker\z/i, "")
50
+ end
46
51
 
52
+ def test_framework
53
+ ::Rails.application.config.generators.options[:rails][:test_framework]
54
+ end
47
55
  end
48
56
  end
49
57
  end