sidekiq 4.2.10 → 6.1.2
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/.github/ISSUE_TEMPLATE/bug_report.md +20 -0
- data/.github/workflows/ci.yml +41 -0
- data/.gitignore +2 -1
- data/.standard.yml +20 -0
- data/5.0-Upgrade.md +56 -0
- data/6.0-Upgrade.md +72 -0
- data/COMM-LICENSE +12 -10
- data/Changes.md +354 -1
- data/Ent-2.0-Upgrade.md +37 -0
- data/Ent-Changes.md +111 -3
- data/Gemfile +16 -21
- data/Gemfile.lock +192 -0
- data/LICENSE +1 -1
- data/Pro-4.0-Upgrade.md +35 -0
- data/Pro-5.0-Upgrade.md +25 -0
- data/Pro-Changes.md +181 -4
- data/README.md +19 -33
- data/Rakefile +6 -8
- data/bin/sidekiq +26 -2
- data/bin/sidekiqload +37 -34
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +1 -1
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +1 -1
- data/lib/generators/sidekiq/worker_generator.rb +21 -13
- data/lib/sidekiq.rb +86 -61
- data/lib/sidekiq/api.rb +320 -209
- data/lib/sidekiq/cli.rb +207 -217
- data/lib/sidekiq/client.rb +78 -51
- data/lib/sidekiq/delay.rb +41 -0
- data/lib/sidekiq/exception_handler.rb +12 -16
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +10 -4
- data/lib/sidekiq/fetch.rb +29 -30
- data/lib/sidekiq/job_logger.rb +63 -0
- data/lib/sidekiq/job_retry.rb +262 -0
- data/lib/sidekiq/launcher.rb +102 -69
- data/lib/sidekiq/logger.rb +165 -0
- data/lib/sidekiq/manager.rb +16 -19
- data/lib/sidekiq/middleware/chain.rb +15 -5
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +18 -14
- data/lib/sidekiq/processor.rb +161 -82
- data/lib/sidekiq/rails.rb +27 -100
- data/lib/sidekiq/redis_connection.rb +60 -20
- data/lib/sidekiq/scheduled.rb +61 -35
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing.rb +48 -28
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/util.rb +20 -16
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web.rb +57 -57
- data/lib/sidekiq/web/action.rb +14 -14
- data/lib/sidekiq/web/application.rb +103 -84
- data/lib/sidekiq/web/csrf_protection.rb +158 -0
- data/lib/sidekiq/web/helpers.rb +126 -71
- data/lib/sidekiq/web/router.rb +18 -17
- data/lib/sidekiq/worker.rb +164 -41
- data/sidekiq.gemspec +15 -27
- data/web/assets/javascripts/application.js +25 -27
- data/web/assets/javascripts/dashboard.js +33 -37
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +246 -0
- data/web/assets/stylesheets/application.css +385 -10
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +81 -0
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +4 -0
- data/web/locales/es.yml +4 -3
- data/web/locales/fa.yml +1 -0
- data/web/locales/fr.yml +2 -2
- data/web/locales/he.yml +79 -0
- data/web/locales/ja.yml +9 -4
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/locales/ur.yml +80 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +2 -1
- data/web/views/_nav.erb +4 -18
- data/web/views/_paging.erb +1 -1
- data/web/views/busy.erb +15 -8
- data/web/views/dashboard.erb +1 -1
- data/web/views/dead.erb +2 -2
- data/web/views/layout.erb +12 -2
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +18 -8
- data/web/views/queues.erb +11 -1
- data/web/views/retries.erb +14 -7
- data/web/views/retry.erb +2 -2
- data/web/views/scheduled.erb +7 -4
- metadata +41 -188
- data/.github/issue_template.md +0 -9
- data/.travis.yml +0 -18
- data/bin/sidekiqctl +0 -99
- data/lib/sidekiq/core_ext.rb +0 -119
- data/lib/sidekiq/logging.rb +0 -106
- data/lib/sidekiq/middleware/server/active_record.rb +0 -13
- data/lib/sidekiq/middleware/server/logging.rb +0 -31
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -205
data/Pro-Changes.md
CHANGED
@@ -1,11 +1,188 @@
|
|
1
|
-
Sidekiq Pro Changelog
|
2
|
-
=======================
|
1
|
+
# Sidekiq Pro Changelog
|
3
2
|
|
4
|
-
|
3
|
+
[Sidekiq Changes](https://github.com/mperham/sidekiq/blob/master/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/master/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/master/Ent-Changes.md)
|
5
4
|
|
6
|
-
|
5
|
+
Please see [sidekiq.org](https://sidekiq.org/) for more details and how to buy.
|
6
|
+
|
7
|
+
5.2.0
|
8
|
+
---------
|
9
|
+
|
10
|
+
- The Sidekiq Pro and Enterprise gem servers now `bundle install` much faster with **Bundler 2.2+** [#4158]
|
11
|
+
- Fix issue with reliable push and multiple shards [#4669]
|
12
|
+
- Fix Pro memory leak due to fetch refactoring in Sidekiq 6.1 [#4652]
|
13
|
+
- Gracefully handle poison pill jobs [#4633]
|
14
|
+
- Remove support for multi-shard batches [#4642]
|
15
|
+
- Rename `Sidekiq::Rack::BatchStatus` to `Sidekiq::Pro::BatchStatus` [#4655]
|
16
|
+
|
17
|
+
5.1.1
|
18
|
+
---------
|
19
|
+
|
20
|
+
- Fix broken basic fetcher [#4616]
|
21
|
+
|
22
|
+
5.1.0
|
23
|
+
---------
|
24
|
+
|
25
|
+
- Remove old Statsd metrics with `WorkerName` in the name [#4377]
|
26
|
+
```
|
27
|
+
job.WorkerName.count -> job.count with tag worker:WorkerName
|
28
|
+
job.WorkerName.perform -> job.perform with tag worker:WorkerName
|
29
|
+
job.WorkerName.failure -> job.failure with tag worker:WorkerName
|
30
|
+
```
|
31
|
+
- Remove `concurrent-ruby` gem dependency [#4586]
|
32
|
+
- Update `constantize` for batch callbacks. [#4469]
|
33
|
+
- Add queue tag to `jobs.recovered.fetch` metric [#4594]
|
34
|
+
- Refactor Pro's fetch infrastructure [#4602]
|
35
|
+
|
36
|
+
5.0.1
|
37
|
+
---------
|
38
|
+
|
39
|
+
- Rejigger batch failures UI to add direct links to retries and scheduled jobs [#4209]
|
40
|
+
- Delete batch data with `UNLINK` [#4155]
|
41
|
+
- Fix bug where a scheduled job can lose its scheduled time when using reliable push [#4267]
|
42
|
+
- Sidekiq::JobSet#scan and #find_job APIs have been promoted to Sidekiq OSS. [#4259]
|
43
|
+
|
44
|
+
5.0.0
|
45
|
+
---------
|
46
|
+
|
47
|
+
- There is no significant migration from Sidekiq Pro 4.0 to 5.0
|
48
|
+
but make sure you read the [update notes for Sidekiq
|
49
|
+
6.0](https://github.com/mperham/sidekiq/blob/master/6.0-Upgrade.md).
|
50
|
+
- Removed various deprecated APIs and associated warnings.
|
51
|
+
- **BREAKING CHANGE** Remove the `Sidekiq::Batch::Status#dead_jobs` API in favor of
|
52
|
+
`Sidekiq::Batch::Status#dead_jids`. [#4217]
|
53
|
+
- Update Sidekiq Pro codebase to use StandardRB formatting
|
54
|
+
- Fix lingering "b-XXX-died" elements in Redis which could cause
|
55
|
+
excessive memory usage. [#4217]
|
56
|
+
- Add ES translations, see issues [#3949](https://github.com/mperham/sidekiq/issues/3949) and [#3951](https://github.com/mperham/sidekiq/issues/3951) to add your own language.
|
57
|
+
|
58
|
+
4.0.5
|
59
|
+
---------
|
60
|
+
|
61
|
+
- Increase super\_fetch retriever thread count from 1 to 2 to make it
|
62
|
+
less sensitive to Redis latency.
|
63
|
+
- Better handling of invalid job JSON by reliable scheduler [#4053]
|
64
|
+
- Added ZH, PT, JA and RU translations.
|
65
|
+
|
66
|
+
4.0.4
|
67
|
+
---------
|
68
|
+
|
69
|
+
- Update Sidekiq::Client patches to work with new Module#prepend
|
70
|
+
mechanism in Sidekiq 5.2.0. [#3930]
|
71
|
+
|
72
|
+
4.0.3
|
73
|
+
---------
|
74
|
+
|
75
|
+
- Add at\_exit handler to push any saved jobs in `reliable_push` when exiting. [#3823]
|
76
|
+
- Implement batch death callback. This is fired the first time a job within a batch dies. [#3841]
|
77
|
+
```ruby
|
78
|
+
batch = Sidekiq::Batch.new
|
79
|
+
batch.on(:death, ...)
|
80
|
+
```
|
81
|
+
|
82
|
+
4.0.2
|
83
|
+
---------
|
84
|
+
|
85
|
+
- Remove super\_fetch edge case leading to an unnecessary `sleep(1)`
|
86
|
+
call and resulting latency [#3790]
|
87
|
+
- Fix possible bad statsd metric call on super\_fetch startup
|
88
|
+
- Remove superfluous `freeze` calls on Strings [#3759]
|
89
|
+
|
90
|
+
4.0.1
|
91
|
+
---------
|
92
|
+
|
93
|
+
- Fix incompatibility with the statsd-ruby gem [#3740]
|
94
|
+
- Add tags to Statsd metrics when using Datadog [#3744]
|
95
|
+
|
96
|
+
4.0.0
|
97
|
+
---------
|
98
|
+
|
99
|
+
- See the [Sidekiq Pro 4.0](Pro-4.0-Upgrade.md) release notes.
|
100
|
+
|
101
|
+
|
102
|
+
3.7.1
|
103
|
+
---------
|
104
|
+
|
105
|
+
- Deprecate timed\_fetch. Switch to super\_fetch:
|
106
|
+
```ruby
|
107
|
+
config.super_fetch!
|
108
|
+
```
|
109
|
+
|
110
|
+
|
111
|
+
3.7.0
|
112
|
+
---------
|
113
|
+
|
114
|
+
- Refactor batch job success/failure to gracefully handle several edge
|
115
|
+
cases with regard to Sidekiq::Shutdown. This should greatly reduce
|
116
|
+
the chances of seeing the long-standing "negative pending count" problem. [#3710]
|
117
|
+
|
118
|
+
|
119
|
+
3.6.1
|
120
|
+
---------
|
121
|
+
|
122
|
+
- Add support for Datadog::Statsd, it is the recommended Statsd client. [#3699]
|
123
|
+
```ruby
|
124
|
+
Sidekiq::Pro.dogstatsd = ->{ Datadog::Statsd.new("metrics.example.com", 8125) }
|
125
|
+
```
|
126
|
+
- Size the statsd connection pool based on Sidekiq's concurrency [#3700]
|
127
|
+
|
128
|
+
|
129
|
+
3.6.0
|
130
|
+
---------
|
131
|
+
|
132
|
+
This release overhauls the Statsd metrics support and adds more
|
133
|
+
metrics for tracking Pro feature usage. In your initializer:
|
134
|
+
```ruby
|
135
|
+
Sidekiq::Pro.statsd = ->{ ::Statsd.new("127.0.0.1", 8125) }
|
136
|
+
```
|
137
|
+
Sidekiq Pro will emit more metrics to Statsd:
|
138
|
+
```
|
139
|
+
jobs.expired - when a job is expired
|
140
|
+
jobs.recovered.push - when a job is recovered by reliable_push after network outage
|
141
|
+
jobs.recovered.fetch - when a job is recovered by super_fetch after process crash
|
142
|
+
batch.created - when a batch is created
|
143
|
+
batch.complete - when a batch is completed
|
144
|
+
batch.success - when a batch is successful
|
145
|
+
```
|
146
|
+
Sidekiq Pro's existing Statsd middleware has been rewritten to leverage the new API.
|
147
|
+
Everything should be backwards compatible with one deprecation notice.
|
148
|
+
|
149
|
+
|
150
|
+
3.5.4
|
151
|
+
---------
|
152
|
+
|
153
|
+
- Fix case in SuperFetch where Redis downtime can lead to processor thread death [#3684]
|
154
|
+
- Fix case where TimedFetch might not recover some pending jobs
|
155
|
+
- Fix edge case in Batch::Status#poll leading to premature completion [#3640]
|
156
|
+
- Adjust scan API to check 100 elements at a time, to minimize network round trips
|
157
|
+
when scanning large sets.
|
158
|
+
|
159
|
+
3.5.3
|
160
|
+
---------
|
161
|
+
|
162
|
+
- Restore error check for super\_fetch's job ack [#3601]
|
163
|
+
- Trim error messages saved in Batch's failure hash, preventing huge
|
164
|
+
messages from bloating Redis. [#3570]
|
165
|
+
|
166
|
+
3.5.2
|
167
|
+
---------
|
168
|
+
|
169
|
+
- Fix `Status#completed?` when run against a Batch that had succeeded
|
170
|
+
and was deleted. [#3519]
|
171
|
+
|
172
|
+
3.5.1
|
173
|
+
---------
|
174
|
+
|
175
|
+
- Work with Sidekiq 5.0.2+
|
176
|
+
- Improve performance of super\_fetch with weighted queues [#3489]
|
177
|
+
|
178
|
+
3.5.0
|
7
179
|
---------
|
8
180
|
|
181
|
+
- Add queue pause/unpause endpoints for scripting via curl [#3445]
|
182
|
+
- Change how super\_fetch names private queues to avoid hostname/queue clashes. [#3443]
|
183
|
+
- Re-implement `Sidekiq::Queue#delete_job` to avoid O(n) runtime [#3408]
|
184
|
+
- Batch page displays Pending JIDs if less than 10 [#3130]
|
185
|
+
- Batch page has a Search button to find associated Retries [#3130]
|
9
186
|
- Make Batch UI progress bar more friendly to the colorblind [#3387]
|
10
187
|
|
11
188
|
3.4.5
|
data/README.md
CHANGED
@@ -2,10 +2,7 @@ Sidekiq
|
|
2
2
|
==============
|
3
3
|
|
4
4
|
[![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
|
5
|
-
|
6
|
-
[![Build Status](https://travis-ci.org/mperham/sidekiq.svg)](https://travis-ci.org/mperham/sidekiq)
|
7
|
-
[![Gitter Chat](https://badges.gitter.im/mperham/sidekiq.svg)](https://gitter.im/mperham/sidekiq)
|
8
|
-
|
5
|
+
![Build](https://github.com/mperham/sidekiq/workflows/CI/badge.svg)
|
9
6
|
|
10
7
|
Simple, efficient background processing for Ruby.
|
11
8
|
|
@@ -13,31 +10,27 @@ Sidekiq uses threads to handle many jobs at the same time in the
|
|
13
10
|
same process. It does not require Rails but will integrate tightly with
|
14
11
|
Rails to make background processing dead simple.
|
15
12
|
|
16
|
-
Sidekiq is compatible with Resque. It uses the exact same
|
17
|
-
message format as Resque so it can integrate into an existing Resque processing farm.
|
18
|
-
You can have Sidekiq and Resque run side-by-side at the same time and
|
19
|
-
use the Resque client to enqueue jobs in Redis to be processed by Sidekiq.
|
20
|
-
|
21
13
|
Performance
|
22
14
|
---------------
|
23
15
|
|
24
|
-
Version | Latency | Garbage created for
|
25
|
-
|
26
|
-
Sidekiq
|
27
|
-
Sidekiq
|
28
|
-
|
29
|
-
|
16
|
+
Version | Latency | Garbage created for 10k jobs | Time to process 100k jobs | Throughput | Ruby
|
17
|
+
-----------------|------|---------|---------|------------------------|-----
|
18
|
+
Sidekiq 6.0.2 | 3 ms | 156 MB | 14.0 sec| **7100 jobs/sec** | MRI 2.6.3
|
19
|
+
Sidekiq 6.0.0 | 3 ms | 156 MB | 19 sec | 5200 jobs/sec | MRI 2.6.3
|
20
|
+
Sidekiq 4.0.0 | 10 ms | 151 MB | 22 sec | 4500 jobs/sec |
|
21
|
+
Sidekiq 3.5.1 | 22 ms | 1257 MB | 125 sec | 800 jobs/sec |
|
22
|
+
Resque 1.25.2 | - | - | 420 sec | 240 jobs/sec |
|
23
|
+
DelayedJob 4.1.1 | - | - | 465 sec | 215 jobs/sec |
|
30
24
|
|
25
|
+
This benchmark can be found in `bin/sidekiqload` and assumes a Redis network latency of 1ms.
|
31
26
|
|
32
27
|
Requirements
|
33
28
|
-----------------
|
34
29
|
|
35
|
-
|
30
|
+
- Redis: 4.0+
|
31
|
+
- Ruby: MRI 2.5+ or JRuby 9.2+.
|
36
32
|
|
37
|
-
|
38
|
-
|
39
|
-
Redis 2.8 or greater is required. 3.0.3+ is recommended for large
|
40
|
-
installations with thousands of worker threads.
|
33
|
+
Sidekiq 6.0 supports Rails 5.0+ but does not require it.
|
41
34
|
|
42
35
|
|
43
36
|
Installation
|
@@ -62,7 +55,7 @@ Want to Upgrade?
|
|
62
55
|
I also sell Sidekiq Pro and Sidekiq Enterprise, extensions to Sidekiq which provide more
|
63
56
|
features, a commercial-friendly license and allow you to support high
|
64
57
|
quality open source development all at the same time. Please see the
|
65
|
-
[Sidekiq](
|
58
|
+
[Sidekiq](https://sidekiq.org/) homepage for more detail.
|
66
59
|
|
67
60
|
Subscribe to the **[quarterly newsletter](https://tinyletter.com/sidekiq)** to stay informed about the latest
|
68
61
|
features and changes to Sidekiq and its bigger siblings.
|
@@ -76,23 +69,16 @@ Problems?
|
|
76
69
|
If you have a problem, please review the [FAQ](https://github.com/mperham/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/mperham/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages.
|
77
70
|
Searching the [issues](https://github.com/mperham/sidekiq/issues) for your problem is also a good idea.
|
78
71
|
|
72
|
+
Sidekiq Pro and Sidekiq Enterprise customers get private email support. You can purchase at https://sidekiq.org; email support@contribsys.com for help.
|
73
|
+
|
79
74
|
Useful resources:
|
80
75
|
|
81
76
|
* Product documentation is in the [wiki](https://github.com/mperham/sidekiq/wiki).
|
82
|
-
*
|
83
|
-
* Here's a [Reddit forum](https://reddit.com/r/sidekiq) dedicated to Sidekiq discussion
|
77
|
+
* Occasional announcements are made to the [@sidekiq](https://twitter.com/sidekiq) Twitter account.
|
84
78
|
* The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q & A.
|
85
79
|
|
86
|
-
**No support via Twitter, 140 characters is not enough.**
|
87
|
-
|
88
80
|
Every Friday morning is Sidekiq happy hour: I video chat and answer questions.
|
89
|
-
See the [Sidekiq support page](
|
90
|
-
|
91
|
-
Thanks
|
92
|
-
-----------------
|
93
|
-
|
94
|
-
Sidekiq stays fast by using the [JProfiler java profiler](http://www.ej-technologies.com/products/jprofiler/overview.html) to find and fix
|
95
|
-
performance problems on JRuby. Unfortunately MRI does not have good multithreaded profiling tools.
|
81
|
+
See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
|
96
82
|
|
97
83
|
|
98
84
|
License
|
@@ -104,4 +90,4 @@ Please see [LICENSE](https://github.com/mperham/sidekiq/blob/master/LICENSE) for
|
|
104
90
|
Author
|
105
91
|
-----------------
|
106
92
|
|
107
|
-
Mike Perham, [@
|
93
|
+
Mike Perham, [@getajobmike](https://twitter.com/getajobmike) / [@sidekiq](https://twitter.com/sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
|
data/Rakefile
CHANGED
@@ -1,12 +1,10 @@
|
|
1
|
-
require
|
2
|
-
require
|
1
|
+
require "bundler/gem_tasks"
|
2
|
+
require "rake/testtask"
|
3
|
+
require "standard/rake"
|
4
|
+
|
3
5
|
Rake::TestTask.new(:test) do |test|
|
4
6
|
test.warning = true
|
5
|
-
test.pattern =
|
7
|
+
test.pattern = "test/**/test_*.rb"
|
6
8
|
end
|
7
9
|
|
8
|
-
task :
|
9
|
-
|
10
|
-
task :appraise do
|
11
|
-
exec("cd myapp && rake appraise")
|
12
|
-
end
|
10
|
+
task default: [:standard, :test]
|
data/bin/sidekiq
CHANGED
@@ -6,13 +6,37 @@ $TESTING = false
|
|
6
6
|
|
7
7
|
require_relative '../lib/sidekiq/cli'
|
8
8
|
|
9
|
+
def integrate_with_systemd
|
10
|
+
return unless ENV["NOTIFY_SOCKET"]
|
11
|
+
|
12
|
+
Sidekiq.configure_server do |config|
|
13
|
+
Sidekiq.logger.info "Enabling systemd notification integration"
|
14
|
+
require "sidekiq/sd_notify"
|
15
|
+
config.on(:startup) do
|
16
|
+
Sidekiq::SdNotify.ready
|
17
|
+
end
|
18
|
+
config.on(:shutdown) do
|
19
|
+
Sidekiq::SdNotify.stopping
|
20
|
+
end
|
21
|
+
Sidekiq.start_watchdog if Sidekiq::SdNotify.watchdog?
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
9
25
|
begin
|
10
26
|
cli = Sidekiq::CLI.instance
|
11
27
|
cli.parse
|
28
|
+
|
29
|
+
integrate_with_systemd
|
30
|
+
|
12
31
|
cli.run
|
13
32
|
rescue => e
|
14
33
|
raise e if $DEBUG
|
15
|
-
|
16
|
-
|
34
|
+
if Sidekiq.error_handlers.length == 0
|
35
|
+
STDERR.puts e.message
|
36
|
+
STDERR.puts e.backtrace.join("\n")
|
37
|
+
else
|
38
|
+
cli.handle_exception e
|
39
|
+
end
|
40
|
+
|
17
41
|
exit 1
|
18
42
|
end
|
data/bin/sidekiqload
CHANGED
@@ -5,7 +5,8 @@
|
|
5
5
|
$TESTING = false
|
6
6
|
|
7
7
|
#require 'ruby-prof'
|
8
|
-
|
8
|
+
require 'bundler/setup'
|
9
|
+
Bundler.require(:default, :load_test)
|
9
10
|
|
10
11
|
require_relative '../lib/sidekiq/cli'
|
11
12
|
require_relative '../lib/sidekiq/launcher'
|
@@ -13,8 +14,8 @@ require_relative '../lib/sidekiq/launcher'
|
|
13
14
|
include Sidekiq::Util
|
14
15
|
|
15
16
|
Sidekiq.configure_server do |config|
|
16
|
-
|
17
|
-
config.redis = { db: 13 }
|
17
|
+
config.options[:concurrency] = 10
|
18
|
+
config.redis = { db: 13, port: 6380, driver: :hiredis }
|
18
19
|
config.options[:queues] << 'default'
|
19
20
|
config.logger.level = Logger::ERROR
|
20
21
|
config.average_scheduled_poll_interval = 2
|
@@ -28,7 +29,8 @@ class LoadWorker
|
|
28
29
|
1
|
29
30
|
end
|
30
31
|
|
31
|
-
def perform(idx)
|
32
|
+
def perform(idx, ts=nil)
|
33
|
+
puts(Time.now.to_f - ts) if ts != nil
|
32
34
|
#raise idx.to_s if idx % 100 == 1
|
33
35
|
end
|
34
36
|
end
|
@@ -36,17 +38,18 @@ end
|
|
36
38
|
# brew tap shopify/shopify
|
37
39
|
# brew install toxiproxy
|
38
40
|
# gem install toxiproxy
|
39
|
-
#
|
41
|
+
# run `toxiproxy-server` in a separate terminal window.
|
42
|
+
require 'toxiproxy'
|
40
43
|
# simulate a non-localhost network for realer-world conditions.
|
41
44
|
# adding 1ms of network latency has an ENORMOUS impact on benchmarks
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
45
|
+
Toxiproxy.populate([{
|
46
|
+
"name": "redis",
|
47
|
+
"listen": "127.0.0.1:6380",
|
48
|
+
"upstream": "127.0.0.1:6379"
|
49
|
+
}])
|
47
50
|
|
48
51
|
self_read, self_write = IO.pipe
|
49
|
-
%w(INT TERM
|
52
|
+
%w(INT TERM TSTP TTIN).each do |sig|
|
50
53
|
begin
|
51
54
|
trap sig do
|
52
55
|
self_write.puts(sig)
|
@@ -65,19 +68,14 @@ def handle_signal(launcher, sig)
|
|
65
68
|
# http://jira.codehaus.org/browse/JRUBY-4637
|
66
69
|
raise Interrupt
|
67
70
|
when 'TERM'
|
68
|
-
# Heroku sends TERM and then waits
|
71
|
+
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
69
72
|
raise Interrupt
|
70
|
-
when '
|
71
|
-
Sidekiq.logger.info "Received
|
73
|
+
when 'TSTP'
|
74
|
+
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
72
75
|
launcher.quiet
|
73
|
-
when 'USR2'
|
74
|
-
if Sidekiq.options[:logfile]
|
75
|
-
Sidekiq.logger.info "Received USR2, reopening log file"
|
76
|
-
Sidekiq::Logging.reopen_logs
|
77
|
-
end
|
78
76
|
when 'TTIN'
|
79
77
|
Thread.list.each do |thread|
|
80
|
-
Sidekiq.logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
|
78
|
+
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['label']}"
|
81
79
|
if thread.backtrace
|
82
80
|
Sidekiq.logger.warn thread.backtrace.join("\n")
|
83
81
|
else
|
@@ -105,21 +103,26 @@ iter.times do
|
|
105
103
|
end
|
106
104
|
Sidekiq.logger.error "Created #{count*iter} jobs"
|
107
105
|
|
106
|
+
start = Time.now
|
107
|
+
|
108
108
|
Monitoring = Thread.new do
|
109
109
|
watchdog("monitor thread") do
|
110
110
|
while true
|
111
|
-
sleep
|
112
|
-
qsize
|
113
|
-
conn.
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
end.map(&:to_i)
|
118
|
-
total = qsize + retries
|
119
|
-
#GC.start
|
120
|
-
Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
|
111
|
+
sleep 0.2
|
112
|
+
qsize = Sidekiq.redis do |conn|
|
113
|
+
conn.llen "queue:default"
|
114
|
+
end
|
115
|
+
total = qsize
|
116
|
+
#Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
|
121
117
|
if total == 0
|
122
|
-
Sidekiq.logger.error("Done")
|
118
|
+
Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
|
119
|
+
Sidekiq.logger.error("Now here's the latency for three jobs")
|
120
|
+
|
121
|
+
LoadWorker.perform_async(1, Time.now.to_f)
|
122
|
+
LoadWorker.perform_async(2, Time.now.to_f)
|
123
|
+
LoadWorker.perform_async(3, Time.now.to_f)
|
124
|
+
|
125
|
+
sleep 0.2
|
123
126
|
exit(0)
|
124
127
|
end
|
125
128
|
end
|
@@ -130,8 +133,8 @@ begin
|
|
130
133
|
#RubyProf::exclude_threads = [ Monitoring ]
|
131
134
|
#RubyProf.start
|
132
135
|
fire_event(:startup)
|
133
|
-
|
134
|
-
|
136
|
+
Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
|
137
|
+
Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
|
135
138
|
launcher = Sidekiq::Launcher.new(Sidekiq.options)
|
136
139
|
launcher.run
|
137
140
|
|
@@ -139,7 +142,7 @@ begin
|
|
139
142
|
signal = readable_io.first[0].gets.strip
|
140
143
|
handle_signal(launcher, signal)
|
141
144
|
end
|
142
|
-
|
145
|
+
end
|
143
146
|
rescue SystemExit => e
|
144
147
|
#Sidekiq.logger.error("Profiling...")
|
145
148
|
#result = RubyProf.stop
|