sidekiq 6.2.0 → 7.0.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (118) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +203 -5
  3. data/LICENSE.txt +9 -0
  4. data/README.md +20 -14
  5. data/bin/sidekiq +4 -9
  6. data/bin/sidekiqload +71 -76
  7. data/bin/sidekiqmon +1 -1
  8. data/lib/generators/sidekiq/job_generator.rb +57 -0
  9. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  10. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  11. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  12. data/lib/sidekiq/api.rb +334 -229
  13. data/lib/sidekiq/capsule.rb +110 -0
  14. data/lib/sidekiq/cli.rb +91 -79
  15. data/lib/sidekiq/client.rb +74 -85
  16. data/lib/sidekiq/{util.rb → component.rb} +13 -14
  17. data/lib/sidekiq/config.rb +271 -0
  18. data/lib/sidekiq/deploy.rb +62 -0
  19. data/lib/sidekiq/embedded.rb +61 -0
  20. data/lib/sidekiq/fetch.rb +26 -24
  21. data/lib/sidekiq/{worker.rb → job.rb} +162 -28
  22. data/lib/sidekiq/job_logger.rb +17 -29
  23. data/lib/sidekiq/job_retry.rb +79 -60
  24. data/lib/sidekiq/job_util.rb +71 -0
  25. data/lib/sidekiq/launcher.rb +96 -96
  26. data/lib/sidekiq/logger.rb +9 -44
  27. data/lib/sidekiq/manager.rb +40 -41
  28. data/lib/sidekiq/metrics/query.rb +153 -0
  29. data/lib/sidekiq/metrics/shared.rb +95 -0
  30. data/lib/sidekiq/metrics/tracking.rb +134 -0
  31. data/lib/sidekiq/middleware/chain.rb +89 -45
  32. data/lib/sidekiq/middleware/current_attributes.rb +58 -0
  33. data/lib/sidekiq/middleware/i18n.rb +6 -4
  34. data/lib/sidekiq/middleware/modules.rb +21 -0
  35. data/lib/sidekiq/monitor.rb +1 -1
  36. data/lib/sidekiq/paginator.rb +16 -8
  37. data/lib/sidekiq/processor.rb +56 -59
  38. data/lib/sidekiq/rails.rb +17 -5
  39. data/lib/sidekiq/redis_client_adapter.rb +118 -0
  40. data/lib/sidekiq/redis_connection.rb +17 -88
  41. data/lib/sidekiq/ring_buffer.rb +29 -0
  42. data/lib/sidekiq/scheduled.rb +102 -39
  43. data/lib/sidekiq/testing/inline.rb +4 -4
  44. data/lib/sidekiq/testing.rb +42 -71
  45. data/lib/sidekiq/transaction_aware_client.rb +44 -0
  46. data/lib/sidekiq/version.rb +2 -1
  47. data/lib/sidekiq/web/action.rb +3 -3
  48. data/lib/sidekiq/web/application.rb +33 -12
  49. data/lib/sidekiq/web/csrf_protection.rb +12 -9
  50. data/lib/sidekiq/web/helpers.rb +30 -40
  51. data/lib/sidekiq/web.rb +9 -19
  52. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  53. data/lib/sidekiq.rb +86 -201
  54. data/sidekiq.gemspec +30 -6
  55. data/web/assets/javascripts/application.js +113 -60
  56. data/web/assets/javascripts/base-charts.js +106 -0
  57. data/web/assets/javascripts/chart.min.js +13 -0
  58. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  59. data/web/assets/javascripts/dashboard-charts.js +166 -0
  60. data/web/assets/javascripts/dashboard.js +36 -273
  61. data/web/assets/javascripts/metrics.js +236 -0
  62. data/web/assets/stylesheets/application-dark.css +28 -45
  63. data/web/assets/stylesheets/application-rtl.css +2 -95
  64. data/web/assets/stylesheets/application.css +84 -529
  65. data/web/locales/ar.yml +71 -65
  66. data/web/locales/cs.yml +62 -62
  67. data/web/locales/da.yml +52 -52
  68. data/web/locales/de.yml +65 -65
  69. data/web/locales/el.yml +43 -24
  70. data/web/locales/en.yml +83 -67
  71. data/web/locales/es.yml +70 -54
  72. data/web/locales/fa.yml +65 -65
  73. data/web/locales/fr.yml +69 -62
  74. data/web/locales/he.yml +65 -64
  75. data/web/locales/hi.yml +59 -59
  76. data/web/locales/it.yml +53 -53
  77. data/web/locales/ja.yml +72 -66
  78. data/web/locales/ko.yml +52 -52
  79. data/web/locales/lt.yml +66 -66
  80. data/web/locales/nb.yml +61 -61
  81. data/web/locales/nl.yml +52 -52
  82. data/web/locales/pl.yml +45 -45
  83. data/web/locales/pt-br.yml +63 -55
  84. data/web/locales/pt.yml +51 -51
  85. data/web/locales/ru.yml +67 -66
  86. data/web/locales/sv.yml +53 -53
  87. data/web/locales/ta.yml +60 -60
  88. data/web/locales/uk.yml +62 -61
  89. data/web/locales/ur.yml +64 -64
  90. data/web/locales/vi.yml +67 -67
  91. data/web/locales/zh-cn.yml +37 -11
  92. data/web/locales/zh-tw.yml +42 -8
  93. data/web/views/_footer.erb +6 -3
  94. data/web/views/_job_info.erb +1 -1
  95. data/web/views/_nav.erb +1 -1
  96. data/web/views/_poll_link.erb +2 -5
  97. data/web/views/_summary.erb +7 -7
  98. data/web/views/busy.erb +17 -11
  99. data/web/views/dashboard.erb +58 -18
  100. data/web/views/dead.erb +1 -1
  101. data/web/views/layout.erb +1 -1
  102. data/web/views/metrics.erb +80 -0
  103. data/web/views/metrics_for_job.erb +69 -0
  104. data/web/views/morgue.erb +6 -6
  105. data/web/views/queue.erb +15 -11
  106. data/web/views/queues.erb +3 -3
  107. data/web/views/retries.erb +7 -7
  108. data/web/views/retry.erb +1 -1
  109. data/web/views/scheduled.erb +1 -1
  110. metadata +78 -28
  111. data/LICENSE +0 -9
  112. data/lib/generators/sidekiq/worker_generator.rb +0 -57
  113. data/lib/sidekiq/delay.rb +0 -41
  114. data/lib/sidekiq/exception_handler.rb +0 -27
  115. data/lib/sidekiq/extensions/action_mailer.rb +0 -48
  116. data/lib/sidekiq/extensions/active_record.rb +0 -43
  117. data/lib/sidekiq/extensions/class_methods.rb +0 -43
  118. data/lib/sidekiq/extensions/generic_proxy.rb +0 -31
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 990d3e553aed906265ffa0cafb74fa4c79e0ccde957f21ccc3d09531d01e75bf
4
- data.tar.gz: 92a68ab1ea824dc78b91610e98c3d10ce4a5794e7aebd66803dc6f29db419420
3
+ metadata.gz: 9c6762c97172b3f8e4b7cc1fd940756ee8796faf70ece8d5e9ede9e2dc7797fe
4
+ data.tar.gz: 905e0e1f381e1c40832ab2137589161fcccfb63c847ad36c9cca15765d952659
5
5
  SHA512:
6
- metadata.gz: 37664695dd79557af0395fe20855db1d250753dd8c874023a866c27d69e78bfa36b3f70501dad18a6ac64673e62fa8fdf413c77d0052a9ca0d59f25a17df41f9
7
- data.tar.gz: 4d6ef75d6eb4be49e8b8d21d56bd0e300a284a4a6db9a0f0357724206b5e1b81b8bf75b53d8442319fd5fe5a9f08a467d75c59289d82d6aa70a00aa0d5102445
6
+ metadata.gz: acfc9bb74585cc65ad1af27e263668ec154ac791baa99de2a1542cfe462a7b13544a758a7bd574d623088c9e683f705437cc4e4495f05e57770037d030214a44
7
+ data.tar.gz: 31320e6cd2fab0d253f3a32616f51360c4ad6a9ef30acef5f88a6ccc6eaf75e04b76ea12dd053dd29442d96fa49234f65e14801e7600cf0f300c023d75aecf43
data/Changes.md CHANGED
@@ -1,6 +1,193 @@
1
1
  # Sidekiq Changes
2
2
 
3
- [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
3
+ [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
4
+
5
+ main
6
+ ----------
7
+
8
+ - Embedded mode!
9
+ - Capsules!!
10
+ - Job Execution metrics!!!
11
+ - See `docs/7.0-Upgrade.md` for release notes
12
+
13
+ 6-x
14
+ ----------
15
+
16
+ - Add pagination to Busy page [#5556]
17
+ - Speed up WorkSet#each [#5559]
18
+ - Adjust CurrentAttributes to work with the String class name so we aren't referencing
19
+ the Class within a Rails initializer [#5536]
20
+
21
+ 6.5.7
22
+ ----------
23
+
24
+ - Updates for JA and ZH locales
25
+ - Further optimizations for scheduled polling [#5513]
26
+
27
+ 6.5.6
28
+ ----------
29
+
30
+ - Fix deprecation warnings with redis-rb 4.8.0 [#5484]
31
+ - Lock redis-rb to < 5.0 as we are moving to redis-client in Sidekiq 7.0
32
+
33
+ 6.5.5
34
+ ----------
35
+
36
+ - Fix require issue with job_retry.rb [#5462]
37
+ - Improve Sidekiq::Web compatibility with Rack 3.x
38
+
39
+ 6.5.4
40
+ ----------
41
+
42
+ - Fix invalid code on Ruby 2.5 [#5460]
43
+ - Fix further metrics dependency issues [#5457]
44
+
45
+ 6.5.3
46
+ ----------
47
+
48
+ - Don't require metrics code without explicit opt-in [#5456]
49
+
50
+ 6.5.2
51
+ ----------
52
+
53
+ - [Job Metrics are under active development, help wanted!](https://github.com/mperham/sidekiq/wiki/Metrics#contributing) **BETA**
54
+ - Add `Context` column on queue page which shows any CurrentAttributes [#5450]
55
+ - `sidekiq_retry_in` may now return `:discard` or `:kill` to dynamically stop job retries [#5406]
56
+ - Smarter sorting of processes in /busy Web UI [#5398]
57
+ - Fix broken hamburger menu in mobile UI [#5428]
58
+ - Require redis-rb 4.5.0. Note that Sidekiq will break if you use the
59
+ [`Redis.exists_returns_integer = false`](https://github.com/redis/redis-rb/blob/master/CHANGELOG.md#450) flag. [#5394]
60
+
61
+ 6.5.1
62
+ ----------
63
+
64
+ - Fix `push_bulk` breakage [#5387]
65
+
66
+ 6.5.0
67
+ ---------
68
+
69
+ - Substantial refactoring of Sidekiq server internals, part of a larger effort
70
+ to reduce Sidekiq's internal usage of global methods and data, see [docs/global_to_local.md](docs/global_to_local.md) and [docs/middleware.md](docs/middleware.md).
71
+ - **Add beta support for the `redis-client` gem**. This will become the default Redis driver in Sidekiq 7.0. [#5298]
72
+ Read more: https://github.com/mperham/sidekiq/wiki/Using-redis-client
73
+ - **Add beta support for DB transaction-aware client** [#5291]
74
+ Add this line to your initializer and any jobs created during a transaction
75
+ will only be pushed to Redis **after the transaction commits**. You will need to add the
76
+ `after_commit_everywhere` gem to your Gemfile.
77
+ ```ruby
78
+ Sidekiq.transactional_push!
79
+ ```
80
+ This feature does not have a lot of production usage yet; please try it out and let us
81
+ know if you have any issues. It will be fully supported in Sidekiq 7.0 or removed if it
82
+ proves problematic.
83
+ - Fix regression with middleware arguments [#5312]
84
+
85
+ 6.4.2
86
+ ---------
87
+
88
+ - Strict argument checking now runs after client-side middleware [#5246]
89
+ - Fix page events with live polling [#5184]
90
+ - Many under-the-hood changes to remove all usage of the term "worker"
91
+ from the Sidekiq codebase and APIs. This mostly involved RDoc and local
92
+ variable names but a few constants and public APIs were changed. The old
93
+ APIs will be removed in Sidekiq 7.0.
94
+ ```
95
+ Sidekiq::DEFAULT_WORKER_OPTIONS -> Sidekiq.default_job_options
96
+ Sidekiq.default_worker_options -> Sidekiq.default_job_options
97
+ Sidekiq::Queues["default"].jobs_by_worker(HardJob) -> Sidekiq::Queues["default"].jobs_by_class(HardJob)
98
+ ```
99
+
100
+ 6.4.1
101
+ ---------
102
+
103
+ - Fix pipeline/multi deprecations in redis-rb 4.6
104
+ - Fix sidekiq.yml YAML load errors on Ruby 3.1 [#5141]
105
+ - Sharding support for `perform_bulk` [#5129]
106
+ - Refactor job logger for SPEEEEEEED
107
+
108
+ 6.4.0
109
+ ---------
110
+
111
+ - **SECURITY**: Validate input to avoid possible DoS in Web UI.
112
+ - Add **strict argument checking** [#5071]
113
+ Sidekiq will now log a warning if JSON-unsafe arguments are passed to `perform_async`.
114
+ Add `Sidekiq.strict_args!(false)` to your initializer to disable this warning.
115
+ This warning will switch to an exception in Sidekiq 7.0.
116
+ - Note that Delayed Extensions will be removed in Sidekiq 7.0 [#5076]
117
+ - Add `perform_{inline,sync}` in Sidekiq::Job to run a job synchronously [#5061, hasan-ally]
118
+ ```ruby
119
+ SomeJob.perform_async(args...)
120
+ SomeJob.perform_sync(args...)
121
+ SomeJob.perform_inline(args...)
122
+ ```
123
+ You can also dynamically redirect a job to run synchronously:
124
+ ```ruby
125
+ SomeJob.set("sync": true).perform_async(args...) # will run via perform_inline
126
+ ```
127
+ - Replace Sidekiq::Worker `app/workers` generator with Sidekiq::Job `app/sidekiq` generator [#5055]
128
+ ```
129
+ bin/rails generate sidekiq:job ProcessOrderJob
130
+ ```
131
+ - Fix job retries losing CurrentAttributes [#5090]
132
+ - Tweak shutdown to give long-running threads time to cleanup [#5095]
133
+
134
+ 6.3.1
135
+ ---------
136
+
137
+ - Fix keyword arguments error with CurrentAttributes on Ruby 3.0 [#5048]
138
+
139
+ 6.3.0
140
+ ---------
141
+
142
+ - **BREAK**: The Web UI has been refactored to remove jQuery. Any UI extensions
143
+ which use jQuery will break.
144
+ - **FEATURE**: Sidekiq.logger has been enhanced so any `Rails.logger`
145
+ output in jobs now shows up in the Sidekiq console. Remove any logger
146
+ hacks in your initializer and see if it Just Works™ now. [#5021]
147
+ - **FEATURE**: Add `Sidekiq::Job` alias for `Sidekiq::Worker`, to better
148
+ reflect industry standard terminology. You can now do this:
149
+ ```ruby
150
+ class MyJob
151
+ include Sidekiq::Job
152
+ sidekiq_options ...
153
+ def perform(args)
154
+ end
155
+ end
156
+ ```
157
+ - **FEATURE**: Support for serializing ActiveSupport::CurrentAttributes into each job. [#4982]
158
+ ```ruby
159
+ # config/initializers/sidekiq.rb
160
+ require "sidekiq/middleware/current_attributes"
161
+ Sidekiq::CurrentAttributes.persist(Myapp::Current) # Your AS::CurrentAttributes singleton
162
+ ```
163
+ - **FEATURE**: Add `Sidekiq::Worker.perform_bulk` for enqueuing jobs in bulk,
164
+ similar to `Sidekiq::Client.push_bulk` [#5042]
165
+ ```ruby
166
+ MyJob.perform_bulk([[1], [2], [3]])
167
+ ```
168
+ - Implement `queue_as`, `wait` and `wait_until` for ActiveJob compatibility [#5003]
169
+ - Scheduler now uses Lua to reduce Redis load and network roundtrips [#5044]
170
+ - Retry Redis operation if we get an `UNBLOCKED` Redis error [#4985]
171
+ - Run existing signal traps, if any, before running Sidekiq's trap [#4991]
172
+ - Fix fetch bug when using weighted queues which caused Sidekiq to stop
173
+ processing queues randomly [#5031]
174
+
175
+ 6.2.2
176
+ ---------
177
+
178
+ - Reduce retry jitter, add jitter to `sidekiq_retry_in` values [#4957]
179
+ - Minimize scheduler load on Redis at scale [#4882]
180
+ - Improve logging of delay jobs [#4904, BuonOno]
181
+ - Minor CSS improvements for buttons and tables, design PRs always welcome!
182
+ - Tweak Web UI `Cache-Control` header [#4966]
183
+ - Rename internal API class `Sidekiq::Job` to `Sidekiq::JobRecord` [#4955]
184
+
185
+ 6.2.1
186
+ ---------
187
+
188
+ - Update RTT warning logic to handle transient RTT spikes [#4851]
189
+ - Fix very low priority CVE on unescaped queue name [#4852]
190
+ - Add note about sessions and Rails apps in API mode
4
191
 
5
192
  6.2.0
6
193
  ---------
@@ -31,6 +218,10 @@ If this is a bare Rack app, use a session middleware before Sidekiq::Web:
31
218
  # now, update your Rack app to include the secret with a session cookie middleware
32
219
  use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
33
220
  run Sidekiq::Web
221
+
222
+ If this is a Rails app in API mode, you need to enable sessions.
223
+
224
+ https://guides.rubyonrails.org/api_app.html#using-session-middlewares
34
225
  ```
35
226
 
36
227
  6.1.3
@@ -223,6 +414,13 @@ See the [Logging wiki page](https://github.com/mperham/sidekiq/wiki/Logging) for
223
414
  - Integrate the StandardRB code formatter to ensure consistent code
224
415
  styling. [#4114, gearnode]
225
416
 
417
+ 5.2.10
418
+ ---------
419
+
420
+ - Backport fix for CVE-2022-23837.
421
+ - Migrate to `exists?` for redis-rb.
422
+ - Lock redis-rb to <4.6 to avoid deprecations.
423
+
226
424
  5.2.9
227
425
  ---------
228
426
 
@@ -388,7 +586,7 @@ Sidekiq::Middleware::Server::Logging -> Sidekiq::JobLogger
388
586
  - The `SomeWorker.set(options)` API was re-written to avoid thread-local state. [#2152]
389
587
  - Sidekiq Enterprise's encrypted jobs now display "[encrypted data]" in the Web UI instead
390
588
  of random hex bytes.
391
- - Please see the [5.0 Upgrade notes](5.0-Upgrade.md) for more detail.
589
+ - Please see the [5.0 Upgrade notes](docs/5.0-Upgrade.md) for more detail.
392
590
 
393
591
  4.2.10
394
592
  -----------
@@ -606,7 +804,7 @@ Sidekiq::Queues.clear_all
606
804
  - Sidekiq's internals have been completely overhauled for performance
607
805
  and to remove dependencies. This has resulted in major speedups, as
608
806
  [detailed on my blog](http://www.mikeperham.com/2015/10/14/optimizing-sidekiq/).
609
- - See the [4.0 upgrade notes](4.0-Upgrade.md) for more detail.
807
+ - See the [4.0 upgrade notes](docs/4.0-Upgrade.md) for more detail.
610
808
 
611
809
  3.5.4
612
810
  -----------
@@ -873,7 +1071,7 @@ sidekiq_options :dead => false, :retry => 5
873
1071
  3.0.0
874
1072
  -----------
875
1073
 
876
- Please see [3.0-Upgrade.md](3.0-Upgrade.md) for more comprehensive upgrade notes.
1074
+ Please see [3.0-Upgrade.md](docs/3.0-Upgrade.md) for more comprehensive upgrade notes.
877
1075
 
878
1076
  - **Dead Job Queue** - jobs which run out of retries are now moved to a dead
879
1077
  job queue. These jobs must be retried manually or they will expire
@@ -917,7 +1115,7 @@ Sidekiq::Client.via(ConnectionPool.new { Redis.new }) do
917
1115
  end
918
1116
  ```
919
1117
  **Sharding support does require a breaking change to client-side
920
- middleware, see 3.0-Upgrade.md.**
1118
+ middleware, see docs/3.0-Upgrade.md.**
921
1119
  - New Chinese, Greek, Swedish and Czech translations for the Web UI.
922
1120
  - Updated most languages translations for the new UI features.
923
1121
  - **Remove official Capistrano integration** - this integration has been
data/LICENSE.txt ADDED
@@ -0,0 +1,9 @@
1
+ Copyright (c) Contributed Systems LLC
2
+
3
+ Sidekiq is an Open Source project licensed under the terms of
4
+ the LGPLv3 license. Please see <http://www.gnu.org/licenses/lgpl-3.0.html>
5
+ for license text.
6
+
7
+ Sidekiq Pro and Sidekiq Enterprise have a commercial-friendly license.
8
+ You can find the commercial license in COMM-LICENSE.txt.
9
+ Please see https://sidekiq.org for purchasing options.
data/README.md CHANGED
@@ -27,49 +27,51 @@ This benchmark can be found in `bin/sidekiqload` and assumes a Redis network lat
27
27
  Requirements
28
28
  -----------------
29
29
 
30
- - Redis: 4.0+
31
- - Ruby: MRI 2.5+ or JRuby 9.2+.
30
+ - Redis: 6.2+
31
+ - Ruby: MRI 2.7+ or JRuby 9.3+.
32
32
 
33
- Sidekiq 6.0 supports Rails 5.0+ but does not require it.
33
+ Sidekiq 7.0 supports Rails 6.0+ but does not require it.
34
34
 
35
35
 
36
36
  Installation
37
37
  -----------------
38
38
 
39
- gem install sidekiq
39
+ bundle add sidekiq
40
40
 
41
41
 
42
42
  Getting Started
43
43
  -----------------
44
44
 
45
45
  See the [Getting Started wiki page](https://github.com/mperham/sidekiq/wiki/Getting-Started) and follow the simple setup process.
46
- You can watch [this Youtube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
46
+ You can watch [this YouTube playlist](https://www.youtube.com/playlist?list=PLjeHh2LSCFrWGT5uVjUuFKAcrcj5kSai1) to learn all about
47
47
  Sidekiq and see its features in action. Here's the Web UI:
48
48
 
49
- ![Web UI](https://github.com/mperham/sidekiq/raw/master/examples/web-ui.png)
49
+ ![Web UI](https://github.com/mperham/sidekiq/raw/main/examples/web-ui.png)
50
50
 
51
51
 
52
52
  Want to Upgrade?
53
53
  -------------------
54
54
 
55
+ Use `bundle up sidekiq` to upgrade Sidekiq and all its dependencies.
56
+ Upgrade notes between each major version can be found in the `docs/` directory.
57
+
55
58
  I also sell Sidekiq Pro and Sidekiq Enterprise, extensions to Sidekiq which provide more
56
59
  features, a commercial-friendly license and allow you to support high
57
60
  quality open source development all at the same time. Please see the
58
61
  [Sidekiq](https://sidekiq.org/) homepage for more detail.
59
62
 
60
- Subscribe to the **[quarterly newsletter](https://tinyletter.com/sidekiq)** to stay informed about the latest
61
- features and changes to Sidekiq and its bigger siblings.
62
-
63
63
 
64
64
  Problems?
65
65
  -----------------
66
66
 
67
- **Please do not directly email any Sidekiq committers with questions or problems.** A community is best served when discussions are held in public.
67
+ **Please do not directly email any Sidekiq committers with questions or problems.**
68
+ A community is best served when discussions are held in public.
68
69
 
69
70
  If you have a problem, please review the [FAQ](https://github.com/mperham/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/mperham/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages.
70
71
  Searching the [issues](https://github.com/mperham/sidekiq/issues) for your problem is also a good idea.
71
72
 
72
- Sidekiq Pro and Sidekiq Enterprise customers get private email support. You can purchase at https://sidekiq.org; email support@contribsys.com for help.
73
+ Sidekiq Pro and Sidekiq Enterprise customers get private email support.
74
+ You can purchase at https://sidekiq.org; email support@contribsys.com for help.
73
75
 
74
76
  Useful resources:
75
77
 
@@ -77,15 +79,19 @@ Useful resources:
77
79
  * Occasional announcements are made to the [@sidekiq](https://twitter.com/sidekiq) Twitter account.
78
80
  * The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q &amp; A.
79
81
 
80
- Every Friday morning is Sidekiq happy hour: I video chat and answer questions.
82
+ Every Friday morning is Sidekiq office hour: I video chat and answer questions.
81
83
  See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
82
84
 
85
+ Contributing
86
+ -----------------
87
+
88
+ Please see [the contributing guidelines](https://github.com/mperham/sidekiq/blob/main/.github/contributing.md).
83
89
 
84
90
  License
85
91
  -----------------
86
92
 
87
- Please see [LICENSE](https://github.com/mperham/sidekiq/blob/master/LICENSE) for licensing details.
88
-
93
+ Please see [LICENSE.txt](https://github.com/mperham/sidekiq/blob/main/LICENSE.txt) for licensing details.
94
+ The license for Sidekiq Pro and Sidekiq Enterprise can be found in [COMM-LICENSE.txt](https://github.com/mperham/sidekiq/blob/main/COMM-LICENSE.txt).
89
95
 
90
96
  Author
91
97
  -----------------
data/bin/sidekiq CHANGED
@@ -4,13 +4,13 @@
4
4
  # RUBYOPT=-w bundle exec sidekiq
5
5
  $TESTING = false
6
6
 
7
- require_relative '../lib/sidekiq/cli'
7
+ require_relative "../lib/sidekiq/cli"
8
8
 
9
9
  def integrate_with_systemd
10
10
  return unless ENV["NOTIFY_SOCKET"]
11
11
 
12
12
  Sidekiq.configure_server do |config|
13
- Sidekiq.logger.info "Enabling systemd notification integration"
13
+ config.logger.info "Enabling systemd notification integration"
14
14
  require "sidekiq/sd_notify"
15
15
  config.on(:startup) do
16
16
  Sidekiq::SdNotify.ready
@@ -31,12 +31,7 @@ begin
31
31
  cli.run
32
32
  rescue => e
33
33
  raise e if $DEBUG
34
- if Sidekiq.error_handlers.length == 0
35
- STDERR.puts e.message
36
- STDERR.puts e.backtrace.join("\n")
37
- else
38
- cli.handle_exception e
39
- end
40
-
34
+ warn e.message
35
+ warn e.backtrace.join("\n")
41
36
  exit 1
42
37
  end
data/bin/sidekiqload CHANGED
@@ -4,78 +4,71 @@
4
4
  # RUBYOPT=-w bundle exec sidekiq
5
5
  $TESTING = false
6
6
 
7
- #require 'ruby-prof'
8
- require 'bundler/setup'
7
+ # require "ruby-prof"
8
+ require "bundler/setup"
9
9
  Bundler.require(:default, :load_test)
10
10
 
11
- require_relative '../lib/sidekiq/cli'
12
- require_relative '../lib/sidekiq/launcher'
13
-
14
- include Sidekiq::Util
15
-
16
- Sidekiq.configure_server do |config|
17
- config.options[:concurrency] = 10
18
- config.redis = { db: 13, port: 6380, driver: :hiredis }
19
- config.options[:queues] << 'default'
11
+ x = Sidekiq.configure_embed do |config|
12
+ config.redis = {db: 13, port: 6380}
13
+ config.concurrency = 10
14
+ # config.redis = { db: 13, port: 6380, driver: :hiredis}
15
+ config.queues = %w[default]
20
16
  config.logger.level = Logger::ERROR
21
17
  config.average_scheduled_poll_interval = 2
22
18
  config.reliable! if defined?(Sidekiq::Pro)
23
19
  end
24
20
 
25
21
  class LoadWorker
26
- include Sidekiq::Worker
22
+ include Sidekiq::Job
27
23
  sidekiq_options retry: 1
28
24
  sidekiq_retry_in do |x|
29
25
  1
30
26
  end
31
27
 
32
- def perform(idx, ts=nil)
33
- puts(Time.now.to_f - ts) if ts != nil
34
- #raise idx.to_s if idx % 100 == 1
28
+ def perform(idx, ts = nil)
29
+ puts(Time.now.to_f - ts) if !ts.nil?
30
+ # raise idx.to_s if idx % 100 == 1
35
31
  end
36
32
  end
37
33
 
38
34
  # brew tap shopify/shopify
39
35
  # brew install toxiproxy
40
- # gem install toxiproxy
41
36
  # run `toxiproxy-server` in a separate terminal window.
42
- require 'toxiproxy'
37
+ require "toxiproxy"
43
38
  # simulate a non-localhost network for realer-world conditions.
44
39
  # adding 1ms of network latency has an ENORMOUS impact on benchmarks
45
40
  Toxiproxy.populate([{
46
- "name": "redis",
47
- "listen": "127.0.0.1:6380",
48
- "upstream": "127.0.0.1:6379"
41
+ name: "redis",
42
+ listen: "127.0.0.1:6380",
43
+ upstream: "127.0.0.1:6379"
49
44
  }])
50
45
 
51
46
  self_read, self_write = IO.pipe
52
- %w(INT TERM TSTP TTIN).each do |sig|
53
- begin
54
- trap sig do
55
- self_write.puts(sig)
56
- end
57
- rescue ArgumentError
58
- puts "Signal #{sig} not supported"
47
+ %w[INT TERM TSTP TTIN].each do |sig|
48
+ trap sig do
49
+ self_write.puts(sig)
59
50
  end
51
+ rescue ArgumentError
52
+ puts "Signal #{sig} not supported"
60
53
  end
61
54
 
62
- Sidekiq.redis {|c| c.flushdb}
55
+ Sidekiq.redis { |c| c.flushdb }
63
56
  def handle_signal(launcher, sig)
64
57
  Sidekiq.logger.debug "Got #{sig} signal"
65
58
  case sig
66
- when 'INT'
59
+ when "INT"
67
60
  # Handle Ctrl-C in JRuby like MRI
68
61
  # http://jira.codehaus.org/browse/JRUBY-4637
69
62
  raise Interrupt
70
- when 'TERM'
63
+ when "TERM"
71
64
  # Heroku sends TERM and then waits 30 seconds for process to exit.
72
65
  raise Interrupt
73
- when 'TSTP'
66
+ when "TSTP"
74
67
  Sidekiq.logger.info "Received TSTP, no longer accepting new work"
75
68
  launcher.quiet
76
- when 'TTIN'
69
+ when "TTIN"
77
70
  Thread.list.each do |thread|
78
- Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['label']}"
71
+ Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["label"]}"
79
72
  if thread.backtrace
80
73
  Sidekiq.logger.warn thread.backtrace.join("\n")
81
74
  else
@@ -93,65 +86,67 @@ iter = 10
93
86
  count = 10_000
94
87
 
95
88
  iter.times do
96
- arr = Array.new(count) do
97
- []
98
- end
99
- count.times do |idx|
100
- arr[idx][0] = idx
101
- end
102
- Sidekiq::Client.push_bulk('class' => LoadWorker, 'args' => arr)
89
+ arr = Array.new(count) { |idx| [idx] }
90
+ Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
103
91
  end
104
- Sidekiq.logger.error "Created #{count*iter} jobs"
92
+ Sidekiq.logger.error "Created #{count * iter} jobs"
105
93
 
106
94
  start = Time.now
107
95
 
108
96
  Monitoring = Thread.new do
109
- watchdog("monitor thread") do
110
- while true
97
+ loop do
98
+ sleep 1.0
99
+ qsize = Sidekiq.redis do |conn|
100
+ conn.llen "queue:default"
101
+ end
102
+ total = qsize
103
+ # Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
104
+ if total == 0
105
+ Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
106
+ Sidekiq.logger.error("Now here's the latency for three jobs")
107
+
108
+ LoadWorker.perform_async(1, Time.now.to_f)
109
+ LoadWorker.perform_async(2, Time.now.to_f)
110
+ LoadWorker.perform_async(3, Time.now.to_f)
111
+
111
112
  sleep 0.2
112
- qsize = Sidekiq.redis do |conn|
113
- conn.llen "queue:default"
114
- end
115
- total = qsize
116
- #Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
117
- if total == 0
118
- Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
119
- Sidekiq.logger.error("Now here's the latency for three jobs")
120
-
121
- LoadWorker.perform_async(1, Time.now.to_f)
122
- LoadWorker.perform_async(2, Time.now.to_f)
123
- LoadWorker.perform_async(3, Time.now.to_f)
124
-
125
- sleep 0.2
126
- exit(0)
127
- end
113
+ exit(0)
128
114
  end
129
115
  end
130
116
  end
131
117
 
118
+ def with_latency(latency, &block)
119
+ Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
120
+ if latency > 0
121
+ Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
122
+ else
123
+ yield
124
+ end
125
+ end
126
+
132
127
  begin
133
- #RubyProf::exclude_threads = [ Monitoring ]
134
- #RubyProf.start
135
- fire_event(:startup)
136
- Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
137
- Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
138
- launcher = Sidekiq::Launcher.new(Sidekiq.options)
139
- launcher.run
140
-
141
- while readable_io = IO.select([self_read])
128
+ # RubyProf.exclude_threads = [Monitoring]
129
+ # RubyProf.start
130
+
131
+ with_latency(Integer(ENV.fetch("LATENCY", "1"))) do
132
+ x.run
133
+
134
+ while (readable_io = IO.select([self_read]))
142
135
  signal = readable_io.first[0].gets.strip
143
- handle_signal(launcher, signal)
136
+ handle_signal(x, signal)
144
137
  end
145
138
  end
146
- rescue SystemExit => e
147
- #Sidekiq.logger.error("Profiling...")
148
- #result = RubyProf.stop
149
- #printer = RubyProf::GraphHtmlPrinter.new(result)
150
- #printer.print(File.new("output.html", "w"), :min_percent => 1)
139
+ rescue SystemExit
140
+ # Sidekiq.logger.error("Profiling...")
141
+ # result = RubyProf.stop
142
+ # printer = RubyProf::GraphHtmlPrinter.new(result)
143
+ # printer.print(File.new("output.html", "w"), min_percent: 1)
151
144
  # normal
152
145
  rescue => e
153
146
  raise e if $DEBUG
154
- STDERR.puts e.message
155
- STDERR.puts e.backtrace.join("\n")
147
+ warn e.message
148
+ warn e.backtrace.join("\n")
156
149
  exit 1
150
+ ensure
151
+ x.stop
157
152
  end
data/bin/sidekiqmon CHANGED
@@ -1,6 +1,6 @@
1
1
  #!/usr/bin/env ruby
2
2
 
3
- require 'sidekiq/monitor'
3
+ require "sidekiq/monitor"
4
4
 
5
5
  section = "all"
6
6
  section = ARGV[0] if ARGV.size == 1
@@ -0,0 +1,57 @@
1
+ require "rails/generators/named_base"
2
+
3
+ module Sidekiq
4
+ module Generators # :nodoc:
5
+ class JobGenerator < ::Rails::Generators::NamedBase # :nodoc:
6
+ desc "This generator creates a Sidekiq Job in app/sidekiq and a corresponding test"
7
+
8
+ check_class_collision suffix: "Job"
9
+
10
+ def self.default_generator_root
11
+ File.dirname(__FILE__)
12
+ end
13
+
14
+ def create_job_file
15
+ template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
16
+ end
17
+
18
+ def create_test_file
19
+ return unless test_framework
20
+
21
+ if test_framework == :rspec
22
+ create_job_spec
23
+ else
24
+ create_job_test
25
+ end
26
+ end
27
+
28
+ private
29
+
30
+ def create_job_spec
31
+ template_file = File.join(
32
+ "spec/sidekiq",
33
+ class_path,
34
+ "#{file_name}_job_spec.rb"
35
+ )
36
+ template "job_spec.rb.erb", template_file
37
+ end
38
+
39
+ def create_job_test
40
+ template_file = File.join(
41
+ "test/sidekiq",
42
+ class_path,
43
+ "#{file_name}_job_test.rb"
44
+ )
45
+ template "job_test.rb.erb", template_file
46
+ end
47
+
48
+ def file_name
49
+ @_file_name ||= super.sub(/_?job\z/i, "")
50
+ end
51
+
52
+ def test_framework
53
+ ::Rails.application.config.generators.options[:rails][:test_framework]
54
+ end
55
+ end
56
+ end
57
+ end