sidekiq 7.3.0 → 8.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Changes.md +158 -0
- data/README.md +16 -13
- data/bin/sidekiqload +31 -22
- data/bin/webload +69 -0
- data/lib/active_job/queue_adapters/sidekiq_adapter.rb +120 -0
- data/lib/generators/sidekiq/job_generator.rb +2 -0
- data/lib/sidekiq/api.rb +184 -71
- data/lib/sidekiq/capsule.rb +11 -9
- data/lib/sidekiq/cli.rb +16 -20
- data/lib/sidekiq/client.rb +28 -11
- data/lib/sidekiq/component.rb +62 -2
- data/lib/sidekiq/config.rb +42 -18
- data/lib/sidekiq/deploy.rb +2 -0
- data/lib/sidekiq/embedded.rb +4 -1
- data/lib/sidekiq/iterable_job.rb +3 -0
- data/lib/sidekiq/job/interrupt_handler.rb +2 -0
- data/lib/sidekiq/job/iterable/active_record_enumerator.rb +3 -3
- data/lib/sidekiq/job/iterable.rb +82 -7
- data/lib/sidekiq/job_logger.rb +15 -27
- data/lib/sidekiq/job_retry.rb +17 -5
- data/lib/sidekiq/job_util.rb +7 -1
- data/lib/sidekiq/launcher.rb +3 -2
- data/lib/sidekiq/logger.rb +19 -70
- data/lib/sidekiq/manager.rb +0 -1
- data/lib/sidekiq/metrics/query.rb +73 -45
- data/lib/sidekiq/metrics/shared.rb +23 -9
- data/lib/sidekiq/metrics/tracking.rb +22 -12
- data/lib/sidekiq/middleware/current_attributes.rb +12 -4
- data/lib/sidekiq/middleware/modules.rb +2 -0
- data/lib/sidekiq/monitor.rb +2 -1
- data/lib/sidekiq/paginator.rb +14 -1
- data/lib/sidekiq/processor.rb +26 -19
- data/lib/sidekiq/profiler.rb +72 -0
- data/lib/sidekiq/rails.rb +44 -55
- data/lib/sidekiq/redis_client_adapter.rb +0 -1
- data/lib/sidekiq/redis_connection.rb +22 -4
- data/lib/sidekiq/ring_buffer.rb +2 -0
- data/lib/sidekiq/systemd.rb +2 -0
- data/lib/sidekiq/testing.rb +7 -7
- data/lib/sidekiq/version.rb +6 -2
- data/lib/sidekiq/web/action.rb +124 -69
- data/lib/sidekiq/web/application.rb +355 -377
- data/lib/sidekiq/web/config.rb +120 -0
- data/lib/sidekiq/web/helpers.rb +64 -33
- data/lib/sidekiq/web/router.rb +61 -74
- data/lib/sidekiq/web.rb +52 -150
- data/lib/sidekiq.rb +5 -4
- data/sidekiq.gemspec +6 -6
- data/web/assets/javascripts/application.js +6 -13
- data/web/assets/javascripts/base-charts.js +30 -16
- data/web/assets/javascripts/chartjs-adapter-date-fns.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +2 -0
- data/web/assets/javascripts/dashboard.js +7 -1
- data/web/assets/javascripts/metrics.js +16 -34
- data/web/assets/stylesheets/style.css +766 -0
- data/web/locales/ar.yml +1 -0
- data/web/locales/cs.yml +1 -0
- data/web/locales/da.yml +1 -0
- data/web/locales/de.yml +1 -0
- data/web/locales/el.yml +1 -0
- data/web/locales/en.yml +9 -1
- data/web/locales/es.yml +24 -2
- data/web/locales/fa.yml +1 -0
- data/web/locales/fr.yml +1 -1
- data/web/locales/gd.yml +1 -1
- data/web/locales/he.yml +1 -0
- data/web/locales/hi.yml +1 -0
- data/web/locales/it.yml +40 -1
- data/web/locales/ja.yml +1 -1
- data/web/locales/ko.yml +1 -0
- data/web/locales/lt.yml +1 -0
- data/web/locales/nb.yml +1 -0
- data/web/locales/nl.yml +1 -0
- data/web/locales/pl.yml +1 -0
- data/web/locales/{pt-br.yml → pt-BR.yml} +3 -3
- data/web/locales/pt.yml +1 -0
- data/web/locales/ru.yml +1 -0
- data/web/locales/sv.yml +1 -0
- data/web/locales/ta.yml +1 -0
- data/web/locales/tr.yml +2 -2
- data/web/locales/uk.yml +25 -1
- data/web/locales/ur.yml +1 -0
- data/web/locales/vi.yml +1 -0
- data/web/locales/{zh-cn.yml → zh-CN.yml} +85 -74
- data/web/locales/{zh-tw.yml → zh-TW.yml} +2 -2
- data/web/views/_footer.erb +31 -34
- data/web/views/_job_info.erb +91 -89
- data/web/views/_metrics_period_select.erb +13 -10
- data/web/views/_nav.erb +14 -21
- data/web/views/_paging.erb +23 -21
- data/web/views/_poll_link.erb +2 -2
- data/web/views/_summary.erb +16 -16
- data/web/views/busy.erb +124 -122
- data/web/views/dashboard.erb +63 -64
- data/web/views/dead.erb +31 -27
- data/web/views/filtering.erb +3 -4
- data/web/views/layout.erb +13 -29
- data/web/views/metrics.erb +75 -82
- data/web/views/metrics_for_job.erb +45 -46
- data/web/views/morgue.erb +61 -70
- data/web/views/profiles.erb +43 -0
- data/web/views/queue.erb +54 -52
- data/web/views/queues.erb +43 -41
- data/web/views/retries.erb +66 -75
- data/web/views/retry.erb +32 -27
- data/web/views/scheduled.erb +59 -55
- data/web/views/scheduled_job_info.erb +1 -1
- metadata +27 -29
- data/web/assets/stylesheets/application-dark.css +0 -147
- data/web/assets/stylesheets/application-rtl.css +0 -163
- data/web/assets/stylesheets/application.css +0 -758
- data/web/assets/stylesheets/bootstrap-rtl.min.css +0 -9
- data/web/assets/stylesheets/bootstrap.css +0 -5
- data/web/views/_status.erb +0 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c6b95635fb1084ed28d8b6dd5f7d273873e24ffab09aeb5057de74d4bebeae07
|
4
|
+
data.tar.gz: bba44aed090637fb4991ae3935112b84b33df8b7281c9d0f273f0146ea11415a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 657b2d2f1e1c850779d29990c88227d7776f8e5a4b3333fd7b2e5f249fb360a1f69de40a773dc5b608ae37ff249713af5aa5b38cd04da2ceffe64f79578861a3
|
7
|
+
data.tar.gz: a8d8d6f32921b00341dc8910155b8a85d6c036bc31a69437dde3f5107d02bed5ffb2b0754f014eaf69f577570e91455a7ead03367cf43fe0b24ef2650e9f57be
|
data/Changes.md
CHANGED
@@ -2,6 +2,164 @@
|
|
2
2
|
|
3
3
|
[Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
|
4
4
|
|
5
|
+
8.0.5
|
6
|
+
----------
|
7
|
+
|
8
|
+
- Add `stopping?` method to AJ adapter for compatibility with the new AJ::Continuations feature [#6732]
|
9
|
+
- Further improvements to Rails boot compatibility [#6710]
|
10
|
+
- Add ability to disable CSRF middleware. SameSite cookies prevent
|
11
|
+
CSRF in a cleaner manner and are default in most browsers now.
|
12
|
+
CSRF code will be removed in Sidekiq 9.0. [#6739]
|
13
|
+
|
14
|
+
8.0.4
|
15
|
+
----------
|
16
|
+
|
17
|
+
- Adjust Rails integration for various edge cases [6713]
|
18
|
+
- Flush job iteration state when an error is raised [#6704]
|
19
|
+
- Update Accept-Language parsing in Web UI [#6721]
|
20
|
+
- Remove fixed-width in Web UI [#6686]
|
21
|
+
- Adjust CSRF middleware ordering [#6688]
|
22
|
+
- Support proxies when POSTing profiles to profiler.firefox.com [#6687]
|
23
|
+
- Dont swallow NoMethodErrors in CurrentAttributes [#6685]
|
24
|
+
|
25
|
+
8.0.3
|
26
|
+
----------
|
27
|
+
|
28
|
+
- Configure Vernier output directory [#6674]
|
29
|
+
- Rework Rails integration [#6669]
|
30
|
+
- Implement flash messages for the Web UI [#6675]
|
31
|
+
|
32
|
+
8.0.2
|
33
|
+
----------
|
34
|
+
|
35
|
+
- Add `on(:exit)` event to run code right before the Sidekiq process exits [#6637]
|
36
|
+
- Metrics page crashes with Rack 3.1+ [#6646]
|
37
|
+
|
38
|
+
8.0.1
|
39
|
+
----------
|
40
|
+
|
41
|
+
- Relax Redis requirement to 7.0 for compatibility with AWS and Ubuntu 24.04 LTS. [#6630]
|
42
|
+
|
43
|
+
8.0.0
|
44
|
+
----------
|
45
|
+
|
46
|
+
- **WARNING** The underlying class name for Active Jobs has changed from `ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper` to `Sidekiq::ActiveJob::Wrapper`.
|
47
|
+
The old name will still work in 8.x.
|
48
|
+
- **WARNING** The `created_at`, `enqueued_at`, `failed_at` and `retried_at` attributes are now stored as epoch milliseconds, rather than epoch floats.
|
49
|
+
This is meant to avoid precision issues with JSON and JavaScript's 53-bit Floats.
|
50
|
+
Example: `"created_at" => 1234567890.123456` -> `"created_at" => 1234567890123`.
|
51
|
+
- **NEW FEATURE** Job Profiling is now supported with [Vernier](https://vernier.prof)
|
52
|
+
which makes it really easy to performance tune your slow jobs.
|
53
|
+
The Web UI contains a new **Profiles** tab to view any collected profile data.
|
54
|
+
Please read the new [Profiling](https://github.com/sidekiq/sidekiq/wiki/Profiling) wiki page for details.
|
55
|
+
- **NEW FEATURE** Job Metrics now store up to 72 hours of data and the Web UI allows display of 24/48/72 hours. [#6614]
|
56
|
+
- CurrentAttribute support now uses `ActiveJob::Arguments` to serialize the context object, supporting Symbols and GlobalID.
|
57
|
+
The change should be backwards compatible. [#6510]
|
58
|
+
- Freshen up `Sidekiq::Web` to simplify the code and improve security [#6532]
|
59
|
+
The CSS has been rewritten from scratch to remove the Bootstrap framework.
|
60
|
+
- Add `on_cancel` callback for iterable jobs [#6607]
|
61
|
+
- Add `cursor` reader to get the current cursor inside iterable jobs [#6606]
|
62
|
+
- Default error logging has been modified to use Ruby's `Exception#detailed_message` and `#full_message` APIs.
|
63
|
+
- CI now runs against Redis, Dragonfly and Valkey.
|
64
|
+
- Job tags now allow custom CSS display [#6595]
|
65
|
+
- The Web UI's language picker now shows options in the native language
|
66
|
+
- Remove global variable usage within the codebase
|
67
|
+
- Colorize and adjust logging for easier reading
|
68
|
+
- Adjust Sidekiq's default thread priority to -1 for a 50ms timeslice.
|
69
|
+
This can help avoid TimeoutErrors when Sidekiq is overloaded. [#6543]
|
70
|
+
- Use `Logger#with_level`, remove Sidekiq's custom impl
|
71
|
+
- Remove `base64` gem dependency
|
72
|
+
- Support: (Dragonfly 1.27+, Valkey 7.2+, Redis 7.2+), Ruby 3.2+, Rails 7.0+
|
73
|
+
|
74
|
+
7.3.10
|
75
|
+
----------
|
76
|
+
|
77
|
+
- Deprecate Redis :password as a String to avoid log disclosure. [#6625]
|
78
|
+
Use a Proc instead: `config.redis = { password: ->(username) { "password" } }`
|
79
|
+
|
80
|
+
7.3.9
|
81
|
+
----------
|
82
|
+
|
83
|
+
- Only require activejob if necessary [#6584]
|
84
|
+
You might get `uninitialized constant Sidekiq::ActiveJob` if you
|
85
|
+
`require 'sidekiq'` before `require 'rails'`.
|
86
|
+
- Fix iterable job cancellation [#6589]
|
87
|
+
- Web UI accessibility improvements [#6604]
|
88
|
+
|
89
|
+
7.3.8
|
90
|
+
----------
|
91
|
+
|
92
|
+
- Fix dead tag links [#6554]
|
93
|
+
- Massive Web UI performance improvement, some pages up to 15x faster [#6555]
|
94
|
+
|
95
|
+
7.3.7
|
96
|
+
----------
|
97
|
+
|
98
|
+
- Backport `Sidekiq::Web.configure` for compatibility with 8.0 [#6532]
|
99
|
+
- Backport `url_params(key)` and `route_params(key)` for compatibility with 8.0 [#6532]
|
100
|
+
- Various fixes for UI filtering [#6508]
|
101
|
+
- Tune `inspect` for internal S::Components to keep size managable [#6553]
|
102
|
+
|
103
|
+
7.3.6
|
104
|
+
----------
|
105
|
+
|
106
|
+
- Forward compatibility fixes for Ruby 3.4
|
107
|
+
- Filtering in the Web UI now works via GET so you can bookmark a filtered view. [#6497]
|
108
|
+
|
109
|
+
7.3.5
|
110
|
+
----------
|
111
|
+
|
112
|
+
- Reimplement `retry_all` and `kill_all` API methods to use ZPOPMIN,
|
113
|
+
approximately 30-60% faster. [#6481]
|
114
|
+
- Add preload testing binary at `examples/testing/sidekiq_boot` to verify your Rails app boots correctly with Sidekiq Enterprise's app preloading.
|
115
|
+
- Fix circular require with ActiveJob adapter [#6477]
|
116
|
+
- Fix potential race condition leading to incorrect serialized values for CurrentAttributes [#6475]
|
117
|
+
- Restore missing elapsed time when default job logging is disabled
|
118
|
+
|
119
|
+
7.3.4
|
120
|
+
----------
|
121
|
+
|
122
|
+
- Fix FrozenError when starting Sidekiq [#6470]
|
123
|
+
|
124
|
+
7.3.3
|
125
|
+
----------
|
126
|
+
|
127
|
+
- Freeze global configuration once boot is complete, to avoid configuration race conditions [#6466, #6465]
|
128
|
+
- Sidekiq now warns if a job iteration takes longer than the `-t` timeout setting (defaults to 25 seconds)
|
129
|
+
- Iteration callbacks now have easy access to job arguments via the `arguments` method:
|
130
|
+
```ruby
|
131
|
+
def on_stop
|
132
|
+
p arguments # => `[123, "string", {"key" => "value"}]`
|
133
|
+
id, str, hash = arguments
|
134
|
+
end
|
135
|
+
```
|
136
|
+
- Iterable jobs can be cancelled via `Sidekiq::Client#cancel!`:
|
137
|
+
```ruby
|
138
|
+
c = Sidekiq::Client.new
|
139
|
+
jid = c.push("class" => SomeJob, "args" => [123])
|
140
|
+
c.cancel!(jid) # => true
|
141
|
+
```
|
142
|
+
- Take over support for ActiveJob's `:sidekiq` adapter [#6430, fatkodima]
|
143
|
+
- Ensure CurrentAttributes are in scope when creating batch callbacks [#6455]
|
144
|
+
- Add `Sidekiq.gem_version` API.
|
145
|
+
- Update Ukranian translations
|
146
|
+
|
147
|
+
7.3.2
|
148
|
+
----------
|
149
|
+
|
150
|
+
- Adjust ActiveRecord batch iteration to restart an interrupted batch from the beginning.
|
151
|
+
Each batch should be processed as a single transaction in order to be idempotent. [#6405]
|
152
|
+
- Fix typo in Sidekiq::DeadSet#kill [#6397]
|
153
|
+
- Fix CSS issue with bottom bar in Web UI [#6414]
|
154
|
+
|
155
|
+
7.3.1
|
156
|
+
----------
|
157
|
+
|
158
|
+
- Don't count job interruptions as failures in metrics [#6386]
|
159
|
+
- Add frozen string literal to a number of .rb files.
|
160
|
+
- Fix frozen string error with style_tag and script_tag [#6371]
|
161
|
+
- Fix an error on Ruby 2.7 because of usage of `Hash#except` [#6376]
|
162
|
+
|
5
163
|
7.3.0
|
6
164
|
----------
|
7
165
|
|
data/README.md
CHANGED
@@ -4,21 +4,23 @@ Sidekiq
|
|
4
4
|
[](https://rubygems.org/gems/sidekiq)
|
5
5
|

|
6
6
|
|
7
|
-
Simple, efficient background
|
7
|
+
Simple, efficient background jobs for Ruby.
|
8
8
|
|
9
9
|
Sidekiq uses threads to handle many jobs at the same time in the
|
10
|
-
same process.
|
11
|
-
Rails to make background processing dead simple.
|
10
|
+
same process. Sidekiq can be used by any Ruby application.
|
12
11
|
|
13
12
|
|
14
13
|
Requirements
|
15
14
|
-----------------
|
16
15
|
|
17
|
-
- Redis: Redis
|
18
|
-
- Ruby: MRI 2
|
16
|
+
- Redis: Redis 7.0+, Valkey 7.2+ or Dragonfly 1.27+
|
17
|
+
- Ruby: MRI 3.2+ or JRuby 9.4+.
|
19
18
|
|
20
|
-
Sidekiq
|
21
|
-
|
19
|
+
Sidekiq 8.0 supports Rails and Active Job 7.0+.
|
20
|
+
|
21
|
+
Sidekiq supports [Valkey](https://valkey.io) and [Dragonfly](https://www.dragonflydb.io) as Redis alternatives.
|
22
|
+
Redis 7.2.4 is considered to be the canonical implementation.
|
23
|
+
Incompatibilities with that version are considered bugs.
|
22
24
|
|
23
25
|
Installation
|
24
26
|
-----------------
|
@@ -42,6 +44,7 @@ The benchmark in `bin/sidekiqload` creates 500,000 no-op jobs and drains them as
|
|
42
44
|
This requires a lot of Redis network I/O and JSON parsing.
|
43
45
|
This benchmark is IO-bound so we increase the concurrency to 25.
|
44
46
|
If your application is sending lots of emails or performing other network-intensive work, you could see a similar benefit but be careful not to saturate the CPU.
|
47
|
+
Real world applications will rarely if ever need to use concurrency greater than 10.
|
45
48
|
|
46
49
|
Version | Time to process 500k jobs | Throughput (jobs/sec) | Ruby | Concurrency | Job Type
|
47
50
|
-----------------|------|---------|---------|------------------------|---
|
@@ -62,7 +65,7 @@ Want to Upgrade?
|
|
62
65
|
Use `bundle up sidekiq` to upgrade Sidekiq and all its dependencies.
|
63
66
|
Upgrade notes between each major version can be found in the `docs/` directory.
|
64
67
|
|
65
|
-
I also sell Sidekiq Pro and Sidekiq Enterprise, extensions to Sidekiq which provide more
|
68
|
+
I also sell [Sidekiq Pro](https://billing.contribsys.com/spro/) and [Sidekiq Enterprise](https://billing.contribsys.com/sent/new.cgi), extensions to Sidekiq which provide more
|
66
69
|
features, a commercial-friendly license and allow you to support high
|
67
70
|
quality open source development all at the same time. Please see the
|
68
71
|
[Sidekiq](https://sidekiq.org/) homepage for more detail.
|
@@ -71,7 +74,7 @@ quality open source development all at the same time. Please see the
|
|
71
74
|
Problems?
|
72
75
|
-----------------
|
73
76
|
|
74
|
-
**
|
77
|
+
**Do not directly email any Sidekiq committers with questions or problems.**
|
75
78
|
A community is best served when discussions are held in public.
|
76
79
|
|
77
80
|
If you have a problem, please review the [FAQ](https://github.com/sidekiq/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/sidekiq/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages.
|
@@ -86,21 +89,21 @@ Useful resources:
|
|
86
89
|
* Occasional announcements are made to the [@sidekiq](https://ruby.social/@sidekiq) Mastodon account.
|
87
90
|
* The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q & A.
|
88
91
|
|
89
|
-
Every Thursday morning is Sidekiq
|
92
|
+
Every Thursday morning is Sidekiq Office Hour: I video chat and answer questions.
|
90
93
|
See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
|
91
94
|
|
92
95
|
Contributing
|
93
96
|
-----------------
|
94
97
|
|
95
|
-
|
98
|
+
See [the contributing guidelines](https://github.com/sidekiq/sidekiq/blob/main/.github/contributing.md).
|
96
99
|
|
97
100
|
License
|
98
101
|
-----------------
|
99
102
|
|
100
|
-
|
103
|
+
See [LICENSE.txt](https://github.com/sidekiq/sidekiq/blob/main/LICENSE.txt) for licensing details.
|
101
104
|
The license for Sidekiq Pro and Sidekiq Enterprise can be found in [COMM-LICENSE.txt](https://github.com/sidekiq/sidekiq/blob/main/COMM-LICENSE.txt).
|
102
105
|
|
103
106
|
Author
|
104
107
|
-----------------
|
105
108
|
|
106
|
-
Mike Perham, [
|
109
|
+
Mike Perham, [mastodon](https://ruby.social/@getajobmike), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
|
data/bin/sidekiqload
CHANGED
@@ -28,7 +28,7 @@ require "ruby-prof" if ENV["PROFILE"]
|
|
28
28
|
require "bundler/setup"
|
29
29
|
Bundler.require(:default, :load_test)
|
30
30
|
|
31
|
-
latency = Integer(ENV["LATENCY"] ||
|
31
|
+
latency = Integer(ENV["LATENCY"] || 0)
|
32
32
|
if latency > 0
|
33
33
|
# brew tap shopify/shopify
|
34
34
|
# brew install toxiproxy
|
@@ -50,7 +50,7 @@ if ENV["AJ"]
|
|
50
50
|
ActiveJob::Base.logger.level = Logger::WARN
|
51
51
|
|
52
52
|
class LoadJob < ActiveJob::Base
|
53
|
-
def perform(idx, ts = nil)
|
53
|
+
def perform(string, idx, hash, ts = nil)
|
54
54
|
puts(Time.now.to_f - ts) if !ts.nil?
|
55
55
|
end
|
56
56
|
end
|
@@ -58,12 +58,21 @@ end
|
|
58
58
|
|
59
59
|
class LoadWorker
|
60
60
|
include Sidekiq::Job
|
61
|
+
$count = 0
|
62
|
+
$lock = Mutex.new
|
63
|
+
|
61
64
|
sidekiq_options retry: 1
|
62
65
|
sidekiq_retry_in do |x|
|
63
66
|
1
|
64
67
|
end
|
65
68
|
|
66
|
-
def perform(idx, ts = nil)
|
69
|
+
def perform(string, idx, hash, ts = nil)
|
70
|
+
$lock.synchronize do
|
71
|
+
$count += 1
|
72
|
+
if $count % 100_000 == 0
|
73
|
+
logger.warn("#{Time.now} Done #{$count}")
|
74
|
+
end
|
75
|
+
end
|
67
76
|
puts(Time.now.to_f - ts) if !ts.nil?
|
68
77
|
# raise idx.to_s if idx % 100 == 1
|
69
78
|
end
|
@@ -77,7 +86,7 @@ class Loader
|
|
77
86
|
def initialize
|
78
87
|
@iter = ENV["GC"] ? 10 : 500
|
79
88
|
@count = Integer(ENV["COUNT"] || 1_000)
|
80
|
-
@latency = Integer(ENV["LATENCY"] ||
|
89
|
+
@latency = Integer(ENV["LATENCY"] || 0)
|
81
90
|
end
|
82
91
|
|
83
92
|
def configure
|
@@ -128,18 +137,18 @@ class Loader
|
|
128
137
|
end
|
129
138
|
|
130
139
|
def setup
|
131
|
-
Sidekiq.logger.
|
140
|
+
Sidekiq.logger.warn("Setup RSS: #{Process.rss}")
|
132
141
|
Sidekiq.redis { |c| c.flushdb }
|
133
142
|
start = Time.now
|
134
143
|
if ENV["AJ"]
|
135
144
|
@iter.times do
|
136
|
-
@count.times do |idx|
|
137
|
-
LoadJob.
|
138
|
-
end
|
145
|
+
ActiveJob.perform_all_later(@count.times.map do |idx|
|
146
|
+
LoadJob.new("mike", idx, {mike: "bob"})
|
147
|
+
end)
|
139
148
|
end
|
140
149
|
else
|
141
150
|
@iter.times do
|
142
|
-
arr = Array.new(@count) { |idx| [idx] }
|
151
|
+
arr = Array.new(@count) { |idx| ["string", idx, {"mike" => "bob"}] }
|
143
152
|
Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
|
144
153
|
end
|
145
154
|
end
|
@@ -158,18 +167,18 @@ class Loader
|
|
158
167
|
if total == 0
|
159
168
|
ending = Time.now - @start
|
160
169
|
size = @iter * @count
|
161
|
-
Sidekiq.logger.
|
162
|
-
Sidekiq.logger.
|
163
|
-
Sidekiq.logger.
|
170
|
+
Sidekiq.logger.warn("Done, #{size} jobs in #{ending} sec, #{(size / ending).to_i} jobs/sec")
|
171
|
+
Sidekiq.logger.warn("Ending RSS: #{Process.rss}")
|
172
|
+
Sidekiq.logger.warn("Now here's the latency for three jobs")
|
164
173
|
|
165
174
|
if ENV["AJ"]
|
166
|
-
LoadJob.perform_later(1, Time.now.to_f)
|
167
|
-
LoadJob.perform_later(2, Time.now.to_f)
|
168
|
-
LoadJob.perform_later(3, Time.now.to_f)
|
175
|
+
LoadJob.perform_later("", 1, {}, Time.now.to_f)
|
176
|
+
LoadJob.perform_later("", 2, {}, Time.now.to_f)
|
177
|
+
LoadJob.perform_later("", 3, {}, Time.now.to_f)
|
169
178
|
else
|
170
|
-
LoadWorker.perform_async(1, Time.now.to_f)
|
171
|
-
LoadWorker.perform_async(2, Time.now.to_f)
|
172
|
-
LoadWorker.perform_async(3, Time.now.to_f)
|
179
|
+
LoadWorker.perform_async("", 1, {}, Time.now.to_f)
|
180
|
+
LoadWorker.perform_async("", 2, {}, Time.now.to_f)
|
181
|
+
LoadWorker.perform_async("", 3, {}, Time.now.to_f)
|
173
182
|
end
|
174
183
|
|
175
184
|
sleep 0.1
|
@@ -182,7 +191,7 @@ class Loader
|
|
182
191
|
end
|
183
192
|
|
184
193
|
def with_latency(latency, &block)
|
185
|
-
Sidekiq.logger.
|
194
|
+
Sidekiq.logger.warn "Simulating #{latency}ms of latency between Sidekiq and redis"
|
186
195
|
if latency > 0
|
187
196
|
Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
|
188
197
|
else
|
@@ -195,8 +204,8 @@ class Loader
|
|
195
204
|
monitor
|
196
205
|
|
197
206
|
if ENV["PROFILE"]
|
198
|
-
RubyProf.exclude_threads
|
199
|
-
|
207
|
+
$profile = RubyProf::Profile.new(exclude_threads: [@monitor])
|
208
|
+
$profile.start
|
200
209
|
elsif ENV["GC"]
|
201
210
|
GC.start
|
202
211
|
GC.compact
|
@@ -227,7 +236,7 @@ class Loader
|
|
227
236
|
Sidekiq.logger.error("GC End RSS: #{Process.rss}") if ENV["GC"]
|
228
237
|
if ENV["PROFILE"]
|
229
238
|
Sidekiq.logger.error("Profiling...")
|
230
|
-
result =
|
239
|
+
result = $profile.stop
|
231
240
|
printer = RubyProf::GraphHtmlPrinter.new(result)
|
232
241
|
printer.print(File.new("output.html", "w"), min_percent: 1)
|
233
242
|
end
|
data/bin/webload
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'bundler'
|
4
|
+
Bundler.setup
|
5
|
+
|
6
|
+
# This skeleton allows you to run Sidekiq::Web page rendering
|
7
|
+
# through Vernier for tuning.
|
8
|
+
require "sidekiq/web"
|
9
|
+
require "rack/test"
|
10
|
+
require "vernier"
|
11
|
+
|
12
|
+
Sidekiq::Web.configure do |config|
|
13
|
+
config.middlewares.clear # remove csrf
|
14
|
+
end
|
15
|
+
|
16
|
+
class SomeJob
|
17
|
+
include Sidekiq::Job
|
18
|
+
end
|
19
|
+
|
20
|
+
class BenchWeb
|
21
|
+
include Rack::Test::Methods
|
22
|
+
|
23
|
+
def app
|
24
|
+
Sidekiq::Web.new
|
25
|
+
end
|
26
|
+
|
27
|
+
def warmup(page = "/scheduled")
|
28
|
+
# Sidekiq.redis {|c| c.flushdb }
|
29
|
+
|
30
|
+
# 100.times do |idx|
|
31
|
+
# SomeJob.perform_at(idx, 1, 3, "mike", {"foo" => "bar"})
|
32
|
+
# end
|
33
|
+
|
34
|
+
100.times do
|
35
|
+
get page
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def load(page = "/scheduled", count = 10_000)
|
40
|
+
profile do
|
41
|
+
count.times do
|
42
|
+
get page
|
43
|
+
raise last_response.inspect unless last_response.status == 200
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def profile(&)
|
49
|
+
if ENV["PROF"]
|
50
|
+
Vernier.profile(out: "profile.json.gz", &)
|
51
|
+
else
|
52
|
+
yield
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def timer(name="block", count = 10_000)
|
58
|
+
a = Time.now
|
59
|
+
yield count
|
60
|
+
b = Time.now
|
61
|
+
puts "#{name} in #{b - a} sec"
|
62
|
+
end
|
63
|
+
|
64
|
+
page = "/busy"
|
65
|
+
b = BenchWeb.new
|
66
|
+
b.warmup(page)
|
67
|
+
timer(page) do |count|
|
68
|
+
b.load(page, count)
|
69
|
+
end
|
@@ -0,0 +1,120 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
begin
|
4
|
+
gem "activejob", ">= 7.0"
|
5
|
+
require "active_job"
|
6
|
+
|
7
|
+
module Sidekiq
|
8
|
+
module ActiveJob
|
9
|
+
# @api private
|
10
|
+
class Wrapper
|
11
|
+
include Sidekiq::Job
|
12
|
+
|
13
|
+
def perform(job_data)
|
14
|
+
::ActiveJob::Base.execute(job_data.merge("provider_job_id" => jid))
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
ActiveSupport.on_load(:active_job) do
|
21
|
+
# By including the Options module, we allow AJs to directly control sidekiq features
|
22
|
+
# via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
|
23
|
+
# AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
|
24
|
+
# manually retried, don't automatically die, etc.
|
25
|
+
#
|
26
|
+
# class SomeJob < ActiveJob::Base
|
27
|
+
# queue_as :default
|
28
|
+
# sidekiq_options retry: 3, backtrace: 10
|
29
|
+
# def perform
|
30
|
+
# end
|
31
|
+
# end
|
32
|
+
include Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
|
33
|
+
end
|
34
|
+
|
35
|
+
# Patch the ActiveJob module
|
36
|
+
module ActiveJob
|
37
|
+
module QueueAdapters
|
38
|
+
# Explicitly remove the implementation existing in older Rails.
|
39
|
+
remove_const(:SidekiqAdapter) if const_defined?(:SidekiqAdapter)
|
40
|
+
|
41
|
+
# Sidekiq adapter for Active Job
|
42
|
+
#
|
43
|
+
# To use Sidekiq set the queue_adapter config to +:sidekiq+.
|
44
|
+
#
|
45
|
+
# Rails.application.config.active_job.queue_adapter = :sidekiq
|
46
|
+
class SidekiqAdapter < AbstractAdapter
|
47
|
+
@@stopping = false
|
48
|
+
|
49
|
+
callback = -> { @@stopping = true }
|
50
|
+
|
51
|
+
Sidekiq.configure_client { |config| config.on(:quiet, &callback) }
|
52
|
+
Sidekiq.configure_server { |config| config.on(:quiet, &callback) }
|
53
|
+
|
54
|
+
# Defines whether enqueuing should happen implicitly to after commit when called
|
55
|
+
# from inside a transaction.
|
56
|
+
# @api private
|
57
|
+
def enqueue_after_transaction_commit?
|
58
|
+
true
|
59
|
+
end
|
60
|
+
|
61
|
+
# @api private
|
62
|
+
def enqueue(job)
|
63
|
+
job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(
|
64
|
+
wrapped: job.class,
|
65
|
+
queue: job.queue_name
|
66
|
+
).perform_async(job.serialize)
|
67
|
+
end
|
68
|
+
|
69
|
+
# @api private
|
70
|
+
def enqueue_at(job, timestamp)
|
71
|
+
job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(
|
72
|
+
wrapped: job.class,
|
73
|
+
queue: job.queue_name
|
74
|
+
).perform_at(timestamp, job.serialize)
|
75
|
+
end
|
76
|
+
|
77
|
+
# @api private
|
78
|
+
def enqueue_all(jobs)
|
79
|
+
enqueued_count = 0
|
80
|
+
jobs.group_by(&:class).each do |job_class, same_class_jobs|
|
81
|
+
same_class_jobs.group_by(&:queue_name).each do |queue, same_class_and_queue_jobs|
|
82
|
+
immediate_jobs, scheduled_jobs = same_class_and_queue_jobs.partition { |job| job.scheduled_at.nil? }
|
83
|
+
|
84
|
+
if immediate_jobs.any?
|
85
|
+
jids = Sidekiq::Client.push_bulk(
|
86
|
+
"class" => Sidekiq::ActiveJob::Wrapper,
|
87
|
+
"wrapped" => job_class,
|
88
|
+
"queue" => queue,
|
89
|
+
"args" => immediate_jobs.map { |job| [job.serialize] }
|
90
|
+
)
|
91
|
+
enqueued_count += jids.compact.size
|
92
|
+
end
|
93
|
+
|
94
|
+
if scheduled_jobs.any?
|
95
|
+
jids = Sidekiq::Client.push_bulk(
|
96
|
+
"class" => Sidekiq::ActiveJob::Wrapper,
|
97
|
+
"wrapped" => job_class,
|
98
|
+
"queue" => queue,
|
99
|
+
"args" => scheduled_jobs.map { |job| [job.serialize] },
|
100
|
+
"at" => scheduled_jobs.map { |job| job.scheduled_at&.to_f }
|
101
|
+
)
|
102
|
+
enqueued_count += jids.compact.size
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
enqueued_count
|
107
|
+
end
|
108
|
+
|
109
|
+
# @api private
|
110
|
+
def stopping? = !!@@stopping
|
111
|
+
|
112
|
+
# Defines a class alias for backwards compatibility with enqueued Active Job jobs.
|
113
|
+
# @api private
|
114
|
+
JobWrapper = Sidekiq::ActiveJob::Wrapper
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
rescue Gem::LoadError
|
119
|
+
# ActiveJob not available or version requirement not met
|
120
|
+
end
|