sidekiq 7.3.0 → 8.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +158 -0
  3. data/README.md +16 -13
  4. data/bin/sidekiqload +31 -22
  5. data/bin/webload +69 -0
  6. data/lib/active_job/queue_adapters/sidekiq_adapter.rb +120 -0
  7. data/lib/generators/sidekiq/job_generator.rb +2 -0
  8. data/lib/sidekiq/api.rb +184 -71
  9. data/lib/sidekiq/capsule.rb +11 -9
  10. data/lib/sidekiq/cli.rb +16 -20
  11. data/lib/sidekiq/client.rb +28 -11
  12. data/lib/sidekiq/component.rb +62 -2
  13. data/lib/sidekiq/config.rb +42 -18
  14. data/lib/sidekiq/deploy.rb +2 -0
  15. data/lib/sidekiq/embedded.rb +4 -1
  16. data/lib/sidekiq/iterable_job.rb +3 -0
  17. data/lib/sidekiq/job/interrupt_handler.rb +2 -0
  18. data/lib/sidekiq/job/iterable/active_record_enumerator.rb +3 -3
  19. data/lib/sidekiq/job/iterable.rb +82 -7
  20. data/lib/sidekiq/job_logger.rb +15 -27
  21. data/lib/sidekiq/job_retry.rb +17 -5
  22. data/lib/sidekiq/job_util.rb +7 -1
  23. data/lib/sidekiq/launcher.rb +3 -2
  24. data/lib/sidekiq/logger.rb +19 -70
  25. data/lib/sidekiq/manager.rb +0 -1
  26. data/lib/sidekiq/metrics/query.rb +73 -45
  27. data/lib/sidekiq/metrics/shared.rb +23 -9
  28. data/lib/sidekiq/metrics/tracking.rb +22 -12
  29. data/lib/sidekiq/middleware/current_attributes.rb +12 -4
  30. data/lib/sidekiq/middleware/modules.rb +2 -0
  31. data/lib/sidekiq/monitor.rb +2 -1
  32. data/lib/sidekiq/paginator.rb +14 -1
  33. data/lib/sidekiq/processor.rb +26 -19
  34. data/lib/sidekiq/profiler.rb +72 -0
  35. data/lib/sidekiq/rails.rb +44 -55
  36. data/lib/sidekiq/redis_client_adapter.rb +0 -1
  37. data/lib/sidekiq/redis_connection.rb +22 -4
  38. data/lib/sidekiq/ring_buffer.rb +2 -0
  39. data/lib/sidekiq/systemd.rb +2 -0
  40. data/lib/sidekiq/testing.rb +7 -7
  41. data/lib/sidekiq/version.rb +6 -2
  42. data/lib/sidekiq/web/action.rb +124 -69
  43. data/lib/sidekiq/web/application.rb +355 -377
  44. data/lib/sidekiq/web/config.rb +120 -0
  45. data/lib/sidekiq/web/helpers.rb +64 -33
  46. data/lib/sidekiq/web/router.rb +61 -74
  47. data/lib/sidekiq/web.rb +52 -150
  48. data/lib/sidekiq.rb +5 -4
  49. data/sidekiq.gemspec +6 -6
  50. data/web/assets/javascripts/application.js +6 -13
  51. data/web/assets/javascripts/base-charts.js +30 -16
  52. data/web/assets/javascripts/chartjs-adapter-date-fns.min.js +7 -0
  53. data/web/assets/javascripts/dashboard-charts.js +2 -0
  54. data/web/assets/javascripts/dashboard.js +7 -1
  55. data/web/assets/javascripts/metrics.js +16 -34
  56. data/web/assets/stylesheets/style.css +766 -0
  57. data/web/locales/ar.yml +1 -0
  58. data/web/locales/cs.yml +1 -0
  59. data/web/locales/da.yml +1 -0
  60. data/web/locales/de.yml +1 -0
  61. data/web/locales/el.yml +1 -0
  62. data/web/locales/en.yml +9 -1
  63. data/web/locales/es.yml +24 -2
  64. data/web/locales/fa.yml +1 -0
  65. data/web/locales/fr.yml +1 -1
  66. data/web/locales/gd.yml +1 -1
  67. data/web/locales/he.yml +1 -0
  68. data/web/locales/hi.yml +1 -0
  69. data/web/locales/it.yml +40 -1
  70. data/web/locales/ja.yml +1 -1
  71. data/web/locales/ko.yml +1 -0
  72. data/web/locales/lt.yml +1 -0
  73. data/web/locales/nb.yml +1 -0
  74. data/web/locales/nl.yml +1 -0
  75. data/web/locales/pl.yml +1 -0
  76. data/web/locales/{pt-br.yml → pt-BR.yml} +3 -3
  77. data/web/locales/pt.yml +1 -0
  78. data/web/locales/ru.yml +1 -0
  79. data/web/locales/sv.yml +1 -0
  80. data/web/locales/ta.yml +1 -0
  81. data/web/locales/tr.yml +2 -2
  82. data/web/locales/uk.yml +25 -1
  83. data/web/locales/ur.yml +1 -0
  84. data/web/locales/vi.yml +1 -0
  85. data/web/locales/{zh-cn.yml → zh-CN.yml} +85 -74
  86. data/web/locales/{zh-tw.yml → zh-TW.yml} +2 -2
  87. data/web/views/_footer.erb +31 -34
  88. data/web/views/_job_info.erb +91 -89
  89. data/web/views/_metrics_period_select.erb +13 -10
  90. data/web/views/_nav.erb +14 -21
  91. data/web/views/_paging.erb +23 -21
  92. data/web/views/_poll_link.erb +2 -2
  93. data/web/views/_summary.erb +16 -16
  94. data/web/views/busy.erb +124 -122
  95. data/web/views/dashboard.erb +63 -64
  96. data/web/views/dead.erb +31 -27
  97. data/web/views/filtering.erb +3 -4
  98. data/web/views/layout.erb +13 -29
  99. data/web/views/metrics.erb +75 -82
  100. data/web/views/metrics_for_job.erb +45 -46
  101. data/web/views/morgue.erb +61 -70
  102. data/web/views/profiles.erb +43 -0
  103. data/web/views/queue.erb +54 -52
  104. data/web/views/queues.erb +43 -41
  105. data/web/views/retries.erb +66 -75
  106. data/web/views/retry.erb +32 -27
  107. data/web/views/scheduled.erb +59 -55
  108. data/web/views/scheduled_job_info.erb +1 -1
  109. metadata +27 -29
  110. data/web/assets/stylesheets/application-dark.css +0 -147
  111. data/web/assets/stylesheets/application-rtl.css +0 -163
  112. data/web/assets/stylesheets/application.css +0 -758
  113. data/web/assets/stylesheets/bootstrap-rtl.min.css +0 -9
  114. data/web/assets/stylesheets/bootstrap.css +0 -5
  115. data/web/views/_status.erb +0 -4
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 34279eae9b8159c48dd31c7b088341136d8b785e7b853389e29e179443af1577
4
- data.tar.gz: 520c6f705995df5c8e50e4e607956bcd8fe78d332e9a33fdf7a4ff25b7534f60
3
+ metadata.gz: c6b95635fb1084ed28d8b6dd5f7d273873e24ffab09aeb5057de74d4bebeae07
4
+ data.tar.gz: bba44aed090637fb4991ae3935112b84b33df8b7281c9d0f273f0146ea11415a
5
5
  SHA512:
6
- metadata.gz: 939c535c352b1c9390e7675d75cfc3728685282187c0edd8475373c6bad1ee67391477a46e26ea3de06724e8cf2570ef759862eb37234c1f9d53f79adafd5947
7
- data.tar.gz: 1a127bf11d514789f6ec808f964d47e9d459687a5b2d91ebe468d6a59a28c5f9dbf01203e1aee583311c776ca9caa41831d74a15557adf5bb0bd71cce27cffdd
6
+ metadata.gz: 657b2d2f1e1c850779d29990c88227d7776f8e5a4b3333fd7b2e5f249fb360a1f69de40a773dc5b608ae37ff249713af5aa5b38cd04da2ceffe64f79578861a3
7
+ data.tar.gz: a8d8d6f32921b00341dc8910155b8a85d6c036bc31a69437dde3f5107d02bed5ffb2b0754f014eaf69f577570e91455a7ead03367cf43fe0b24ef2650e9f57be
data/Changes.md CHANGED
@@ -2,6 +2,164 @@
2
2
 
3
3
  [Sidekiq Changes](https://github.com/sidekiq/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/sidekiq/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/sidekiq/sidekiq/blob/main/Ent-Changes.md)
4
4
 
5
+ 8.0.5
6
+ ----------
7
+
8
+ - Add `stopping?` method to AJ adapter for compatibility with the new AJ::Continuations feature [#6732]
9
+ - Further improvements to Rails boot compatibility [#6710]
10
+ - Add ability to disable CSRF middleware. SameSite cookies prevent
11
+ CSRF in a cleaner manner and are default in most browsers now.
12
+ CSRF code will be removed in Sidekiq 9.0. [#6739]
13
+
14
+ 8.0.4
15
+ ----------
16
+
17
+ - Adjust Rails integration for various edge cases [6713]
18
+ - Flush job iteration state when an error is raised [#6704]
19
+ - Update Accept-Language parsing in Web UI [#6721]
20
+ - Remove fixed-width in Web UI [#6686]
21
+ - Adjust CSRF middleware ordering [#6688]
22
+ - Support proxies when POSTing profiles to profiler.firefox.com [#6687]
23
+ - Dont swallow NoMethodErrors in CurrentAttributes [#6685]
24
+
25
+ 8.0.3
26
+ ----------
27
+
28
+ - Configure Vernier output directory [#6674]
29
+ - Rework Rails integration [#6669]
30
+ - Implement flash messages for the Web UI [#6675]
31
+
32
+ 8.0.2
33
+ ----------
34
+
35
+ - Add `on(:exit)` event to run code right before the Sidekiq process exits [#6637]
36
+ - Metrics page crashes with Rack 3.1+ [#6646]
37
+
38
+ 8.0.1
39
+ ----------
40
+
41
+ - Relax Redis requirement to 7.0 for compatibility with AWS and Ubuntu 24.04 LTS. [#6630]
42
+
43
+ 8.0.0
44
+ ----------
45
+
46
+ - **WARNING** The underlying class name for Active Jobs has changed from `ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper` to `Sidekiq::ActiveJob::Wrapper`.
47
+ The old name will still work in 8.x.
48
+ - **WARNING** The `created_at`, `enqueued_at`, `failed_at` and `retried_at` attributes are now stored as epoch milliseconds, rather than epoch floats.
49
+ This is meant to avoid precision issues with JSON and JavaScript's 53-bit Floats.
50
+ Example: `"created_at" => 1234567890.123456` -> `"created_at" => 1234567890123`.
51
+ - **NEW FEATURE** Job Profiling is now supported with [Vernier](https://vernier.prof)
52
+ which makes it really easy to performance tune your slow jobs.
53
+ The Web UI contains a new **Profiles** tab to view any collected profile data.
54
+ Please read the new [Profiling](https://github.com/sidekiq/sidekiq/wiki/Profiling) wiki page for details.
55
+ - **NEW FEATURE** Job Metrics now store up to 72 hours of data and the Web UI allows display of 24/48/72 hours. [#6614]
56
+ - CurrentAttribute support now uses `ActiveJob::Arguments` to serialize the context object, supporting Symbols and GlobalID.
57
+ The change should be backwards compatible. [#6510]
58
+ - Freshen up `Sidekiq::Web` to simplify the code and improve security [#6532]
59
+ The CSS has been rewritten from scratch to remove the Bootstrap framework.
60
+ - Add `on_cancel` callback for iterable jobs [#6607]
61
+ - Add `cursor` reader to get the current cursor inside iterable jobs [#6606]
62
+ - Default error logging has been modified to use Ruby's `Exception#detailed_message` and `#full_message` APIs.
63
+ - CI now runs against Redis, Dragonfly and Valkey.
64
+ - Job tags now allow custom CSS display [#6595]
65
+ - The Web UI's language picker now shows options in the native language
66
+ - Remove global variable usage within the codebase
67
+ - Colorize and adjust logging for easier reading
68
+ - Adjust Sidekiq's default thread priority to -1 for a 50ms timeslice.
69
+ This can help avoid TimeoutErrors when Sidekiq is overloaded. [#6543]
70
+ - Use `Logger#with_level`, remove Sidekiq's custom impl
71
+ - Remove `base64` gem dependency
72
+ - Support: (Dragonfly 1.27+, Valkey 7.2+, Redis 7.2+), Ruby 3.2+, Rails 7.0+
73
+
74
+ 7.3.10
75
+ ----------
76
+
77
+ - Deprecate Redis :password as a String to avoid log disclosure. [#6625]
78
+ Use a Proc instead: `config.redis = { password: ->(username) { "password" } }`
79
+
80
+ 7.3.9
81
+ ----------
82
+
83
+ - Only require activejob if necessary [#6584]
84
+ You might get `uninitialized constant Sidekiq::ActiveJob` if you
85
+ `require 'sidekiq'` before `require 'rails'`.
86
+ - Fix iterable job cancellation [#6589]
87
+ - Web UI accessibility improvements [#6604]
88
+
89
+ 7.3.8
90
+ ----------
91
+
92
+ - Fix dead tag links [#6554]
93
+ - Massive Web UI performance improvement, some pages up to 15x faster [#6555]
94
+
95
+ 7.3.7
96
+ ----------
97
+
98
+ - Backport `Sidekiq::Web.configure` for compatibility with 8.0 [#6532]
99
+ - Backport `url_params(key)` and `route_params(key)` for compatibility with 8.0 [#6532]
100
+ - Various fixes for UI filtering [#6508]
101
+ - Tune `inspect` for internal S::Components to keep size managable [#6553]
102
+
103
+ 7.3.6
104
+ ----------
105
+
106
+ - Forward compatibility fixes for Ruby 3.4
107
+ - Filtering in the Web UI now works via GET so you can bookmark a filtered view. [#6497]
108
+
109
+ 7.3.5
110
+ ----------
111
+
112
+ - Reimplement `retry_all` and `kill_all` API methods to use ZPOPMIN,
113
+ approximately 30-60% faster. [#6481]
114
+ - Add preload testing binary at `examples/testing/sidekiq_boot` to verify your Rails app boots correctly with Sidekiq Enterprise's app preloading.
115
+ - Fix circular require with ActiveJob adapter [#6477]
116
+ - Fix potential race condition leading to incorrect serialized values for CurrentAttributes [#6475]
117
+ - Restore missing elapsed time when default job logging is disabled
118
+
119
+ 7.3.4
120
+ ----------
121
+
122
+ - Fix FrozenError when starting Sidekiq [#6470]
123
+
124
+ 7.3.3
125
+ ----------
126
+
127
+ - Freeze global configuration once boot is complete, to avoid configuration race conditions [#6466, #6465]
128
+ - Sidekiq now warns if a job iteration takes longer than the `-t` timeout setting (defaults to 25 seconds)
129
+ - Iteration callbacks now have easy access to job arguments via the `arguments` method:
130
+ ```ruby
131
+ def on_stop
132
+ p arguments # => `[123, "string", {"key" => "value"}]`
133
+ id, str, hash = arguments
134
+ end
135
+ ```
136
+ - Iterable jobs can be cancelled via `Sidekiq::Client#cancel!`:
137
+ ```ruby
138
+ c = Sidekiq::Client.new
139
+ jid = c.push("class" => SomeJob, "args" => [123])
140
+ c.cancel!(jid) # => true
141
+ ```
142
+ - Take over support for ActiveJob's `:sidekiq` adapter [#6430, fatkodima]
143
+ - Ensure CurrentAttributes are in scope when creating batch callbacks [#6455]
144
+ - Add `Sidekiq.gem_version` API.
145
+ - Update Ukranian translations
146
+
147
+ 7.3.2
148
+ ----------
149
+
150
+ - Adjust ActiveRecord batch iteration to restart an interrupted batch from the beginning.
151
+ Each batch should be processed as a single transaction in order to be idempotent. [#6405]
152
+ - Fix typo in Sidekiq::DeadSet#kill [#6397]
153
+ - Fix CSS issue with bottom bar in Web UI [#6414]
154
+
155
+ 7.3.1
156
+ ----------
157
+
158
+ - Don't count job interruptions as failures in metrics [#6386]
159
+ - Add frozen string literal to a number of .rb files.
160
+ - Fix frozen string error with style_tag and script_tag [#6371]
161
+ - Fix an error on Ruby 2.7 because of usage of `Hash#except` [#6376]
162
+
5
163
  7.3.0
6
164
  ----------
7
165
 
data/README.md CHANGED
@@ -4,21 +4,23 @@ Sidekiq
4
4
  [![Gem Version](https://badge.fury.io/rb/sidekiq.svg)](https://rubygems.org/gems/sidekiq)
5
5
  ![Build](https://github.com/sidekiq/sidekiq/workflows/CI/badge.svg)
6
6
 
7
- Simple, efficient background processing for Ruby.
7
+ Simple, efficient background jobs for Ruby.
8
8
 
9
9
  Sidekiq uses threads to handle many jobs at the same time in the
10
- same process. It does not require Rails but will integrate tightly with
11
- Rails to make background processing dead simple.
10
+ same process. Sidekiq can be used by any Ruby application.
12
11
 
13
12
 
14
13
  Requirements
15
14
  -----------------
16
15
 
17
- - Redis: Redis 6.2+ or Dragonfly 1.13+
18
- - Ruby: MRI 2.7+ or JRuby 9.3+.
16
+ - Redis: Redis 7.0+, Valkey 7.2+ or Dragonfly 1.27+
17
+ - Ruby: MRI 3.2+ or JRuby 9.4+.
19
18
 
20
- Sidekiq 7.0 supports Rails 6.0+ but does not require it.
21
- As of 7.2, Sidekiq supports Dragonfly as an alternative to Redis for data storage.
19
+ Sidekiq 8.0 supports Rails and Active Job 7.0+.
20
+
21
+ Sidekiq supports [Valkey](https://valkey.io) and [Dragonfly](https://www.dragonflydb.io) as Redis alternatives.
22
+ Redis 7.2.4 is considered to be the canonical implementation.
23
+ Incompatibilities with that version are considered bugs.
22
24
 
23
25
  Installation
24
26
  -----------------
@@ -42,6 +44,7 @@ The benchmark in `bin/sidekiqload` creates 500,000 no-op jobs and drains them as
42
44
  This requires a lot of Redis network I/O and JSON parsing.
43
45
  This benchmark is IO-bound so we increase the concurrency to 25.
44
46
  If your application is sending lots of emails or performing other network-intensive work, you could see a similar benefit but be careful not to saturate the CPU.
47
+ Real world applications will rarely if ever need to use concurrency greater than 10.
45
48
 
46
49
  Version | Time to process 500k jobs | Throughput (jobs/sec) | Ruby | Concurrency | Job Type
47
50
  -----------------|------|---------|---------|------------------------|---
@@ -62,7 +65,7 @@ Want to Upgrade?
62
65
  Use `bundle up sidekiq` to upgrade Sidekiq and all its dependencies.
63
66
  Upgrade notes between each major version can be found in the `docs/` directory.
64
67
 
65
- I also sell Sidekiq Pro and Sidekiq Enterprise, extensions to Sidekiq which provide more
68
+ I also sell [Sidekiq Pro](https://billing.contribsys.com/spro/) and [Sidekiq Enterprise](https://billing.contribsys.com/sent/new.cgi), extensions to Sidekiq which provide more
66
69
  features, a commercial-friendly license and allow you to support high
67
70
  quality open source development all at the same time. Please see the
68
71
  [Sidekiq](https://sidekiq.org/) homepage for more detail.
@@ -71,7 +74,7 @@ quality open source development all at the same time. Please see the
71
74
  Problems?
72
75
  -----------------
73
76
 
74
- **Please do not directly email any Sidekiq committers with questions or problems.**
77
+ **Do not directly email any Sidekiq committers with questions or problems.**
75
78
  A community is best served when discussions are held in public.
76
79
 
77
80
  If you have a problem, please review the [FAQ](https://github.com/sidekiq/sidekiq/wiki/FAQ) and [Troubleshooting](https://github.com/sidekiq/sidekiq/wiki/Problems-and-Troubleshooting) wiki pages.
@@ -86,21 +89,21 @@ Useful resources:
86
89
  * Occasional announcements are made to the [@sidekiq](https://ruby.social/@sidekiq) Mastodon account.
87
90
  * The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q & A.
88
91
 
89
- Every Thursday morning is Sidekiq office hour: I video chat and answer questions.
92
+ Every Thursday morning is Sidekiq Office Hour: I video chat and answer questions.
90
93
  See the [Sidekiq support page](https://sidekiq.org/support.html) for details.
91
94
 
92
95
  Contributing
93
96
  -----------------
94
97
 
95
- Please see [the contributing guidelines](https://github.com/sidekiq/sidekiq/blob/main/.github/contributing.md).
98
+ See [the contributing guidelines](https://github.com/sidekiq/sidekiq/blob/main/.github/contributing.md).
96
99
 
97
100
  License
98
101
  -----------------
99
102
 
100
- Please see [LICENSE.txt](https://github.com/sidekiq/sidekiq/blob/main/LICENSE.txt) for licensing details.
103
+ See [LICENSE.txt](https://github.com/sidekiq/sidekiq/blob/main/LICENSE.txt) for licensing details.
101
104
  The license for Sidekiq Pro and Sidekiq Enterprise can be found in [COMM-LICENSE.txt](https://github.com/sidekiq/sidekiq/blob/main/COMM-LICENSE.txt).
102
105
 
103
106
  Author
104
107
  -----------------
105
108
 
106
- Mike Perham, [@getajobmike](https://ruby.social/@getajobmike) / [@sidekiq](https://ruby.social/@sidekiq), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
109
+ Mike Perham, [mastodon](https://ruby.social/@getajobmike), [https://www.mikeperham.com](https://www.mikeperham.com) / [https://www.contribsys.com](https://www.contribsys.com)
data/bin/sidekiqload CHANGED
@@ -28,7 +28,7 @@ require "ruby-prof" if ENV["PROFILE"]
28
28
  require "bundler/setup"
29
29
  Bundler.require(:default, :load_test)
30
30
 
31
- latency = Integer(ENV["LATENCY"] || 1)
31
+ latency = Integer(ENV["LATENCY"] || 0)
32
32
  if latency > 0
33
33
  # brew tap shopify/shopify
34
34
  # brew install toxiproxy
@@ -50,7 +50,7 @@ if ENV["AJ"]
50
50
  ActiveJob::Base.logger.level = Logger::WARN
51
51
 
52
52
  class LoadJob < ActiveJob::Base
53
- def perform(idx, ts = nil)
53
+ def perform(string, idx, hash, ts = nil)
54
54
  puts(Time.now.to_f - ts) if !ts.nil?
55
55
  end
56
56
  end
@@ -58,12 +58,21 @@ end
58
58
 
59
59
  class LoadWorker
60
60
  include Sidekiq::Job
61
+ $count = 0
62
+ $lock = Mutex.new
63
+
61
64
  sidekiq_options retry: 1
62
65
  sidekiq_retry_in do |x|
63
66
  1
64
67
  end
65
68
 
66
- def perform(idx, ts = nil)
69
+ def perform(string, idx, hash, ts = nil)
70
+ $lock.synchronize do
71
+ $count += 1
72
+ if $count % 100_000 == 0
73
+ logger.warn("#{Time.now} Done #{$count}")
74
+ end
75
+ end
67
76
  puts(Time.now.to_f - ts) if !ts.nil?
68
77
  # raise idx.to_s if idx % 100 == 1
69
78
  end
@@ -77,7 +86,7 @@ class Loader
77
86
  def initialize
78
87
  @iter = ENV["GC"] ? 10 : 500
79
88
  @count = Integer(ENV["COUNT"] || 1_000)
80
- @latency = Integer(ENV["LATENCY"] || 1)
89
+ @latency = Integer(ENV["LATENCY"] || 0)
81
90
  end
82
91
 
83
92
  def configure
@@ -128,18 +137,18 @@ class Loader
128
137
  end
129
138
 
130
139
  def setup
131
- Sidekiq.logger.error("Setup RSS: #{Process.rss}")
140
+ Sidekiq.logger.warn("Setup RSS: #{Process.rss}")
132
141
  Sidekiq.redis { |c| c.flushdb }
133
142
  start = Time.now
134
143
  if ENV["AJ"]
135
144
  @iter.times do
136
- @count.times do |idx|
137
- LoadJob.perform_later(idx)
138
- end
145
+ ActiveJob.perform_all_later(@count.times.map do |idx|
146
+ LoadJob.new("mike", idx, {mike: "bob"})
147
+ end)
139
148
  end
140
149
  else
141
150
  @iter.times do
142
- arr = Array.new(@count) { |idx| [idx] }
151
+ arr = Array.new(@count) { |idx| ["string", idx, {"mike" => "bob"}] }
143
152
  Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
144
153
  end
145
154
  end
@@ -158,18 +167,18 @@ class Loader
158
167
  if total == 0
159
168
  ending = Time.now - @start
160
169
  size = @iter * @count
161
- Sidekiq.logger.error("Done, #{size} jobs in #{ending} sec, #{(size / ending).to_i} jobs/sec")
162
- Sidekiq.logger.error("Ending RSS: #{Process.rss}")
163
- Sidekiq.logger.error("Now here's the latency for three jobs")
170
+ Sidekiq.logger.warn("Done, #{size} jobs in #{ending} sec, #{(size / ending).to_i} jobs/sec")
171
+ Sidekiq.logger.warn("Ending RSS: #{Process.rss}")
172
+ Sidekiq.logger.warn("Now here's the latency for three jobs")
164
173
 
165
174
  if ENV["AJ"]
166
- LoadJob.perform_later(1, Time.now.to_f)
167
- LoadJob.perform_later(2, Time.now.to_f)
168
- LoadJob.perform_later(3, Time.now.to_f)
175
+ LoadJob.perform_later("", 1, {}, Time.now.to_f)
176
+ LoadJob.perform_later("", 2, {}, Time.now.to_f)
177
+ LoadJob.perform_later("", 3, {}, Time.now.to_f)
169
178
  else
170
- LoadWorker.perform_async(1, Time.now.to_f)
171
- LoadWorker.perform_async(2, Time.now.to_f)
172
- LoadWorker.perform_async(3, Time.now.to_f)
179
+ LoadWorker.perform_async("", 1, {}, Time.now.to_f)
180
+ LoadWorker.perform_async("", 2, {}, Time.now.to_f)
181
+ LoadWorker.perform_async("", 3, {}, Time.now.to_f)
173
182
  end
174
183
 
175
184
  sleep 0.1
@@ -182,7 +191,7 @@ class Loader
182
191
  end
183
192
 
184
193
  def with_latency(latency, &block)
185
- Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
194
+ Sidekiq.logger.warn "Simulating #{latency}ms of latency between Sidekiq and redis"
186
195
  if latency > 0
187
196
  Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
188
197
  else
@@ -195,8 +204,8 @@ class Loader
195
204
  monitor
196
205
 
197
206
  if ENV["PROFILE"]
198
- RubyProf.exclude_threads = [@monitor]
199
- RubyProf.start
207
+ $profile = RubyProf::Profile.new(exclude_threads: [@monitor])
208
+ $profile.start
200
209
  elsif ENV["GC"]
201
210
  GC.start
202
211
  GC.compact
@@ -227,7 +236,7 @@ class Loader
227
236
  Sidekiq.logger.error("GC End RSS: #{Process.rss}") if ENV["GC"]
228
237
  if ENV["PROFILE"]
229
238
  Sidekiq.logger.error("Profiling...")
230
- result = RubyProf.stop
239
+ result = $profile.stop
231
240
  printer = RubyProf::GraphHtmlPrinter.new(result)
232
241
  printer.print(File.new("output.html", "w"), min_percent: 1)
233
242
  end
data/bin/webload ADDED
@@ -0,0 +1,69 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'bundler'
4
+ Bundler.setup
5
+
6
+ # This skeleton allows you to run Sidekiq::Web page rendering
7
+ # through Vernier for tuning.
8
+ require "sidekiq/web"
9
+ require "rack/test"
10
+ require "vernier"
11
+
12
+ Sidekiq::Web.configure do |config|
13
+ config.middlewares.clear # remove csrf
14
+ end
15
+
16
+ class SomeJob
17
+ include Sidekiq::Job
18
+ end
19
+
20
+ class BenchWeb
21
+ include Rack::Test::Methods
22
+
23
+ def app
24
+ Sidekiq::Web.new
25
+ end
26
+
27
+ def warmup(page = "/scheduled")
28
+ # Sidekiq.redis {|c| c.flushdb }
29
+
30
+ # 100.times do |idx|
31
+ # SomeJob.perform_at(idx, 1, 3, "mike", {"foo" => "bar"})
32
+ # end
33
+
34
+ 100.times do
35
+ get page
36
+ end
37
+ end
38
+
39
+ def load(page = "/scheduled", count = 10_000)
40
+ profile do
41
+ count.times do
42
+ get page
43
+ raise last_response.inspect unless last_response.status == 200
44
+ end
45
+ end
46
+ end
47
+
48
+ def profile(&)
49
+ if ENV["PROF"]
50
+ Vernier.profile(out: "profile.json.gz", &)
51
+ else
52
+ yield
53
+ end
54
+ end
55
+ end
56
+
57
+ def timer(name="block", count = 10_000)
58
+ a = Time.now
59
+ yield count
60
+ b = Time.now
61
+ puts "#{name} in #{b - a} sec"
62
+ end
63
+
64
+ page = "/busy"
65
+ b = BenchWeb.new
66
+ b.warmup(page)
67
+ timer(page) do |count|
68
+ b.load(page, count)
69
+ end
@@ -0,0 +1,120 @@
1
+ # frozen_string_literal: true
2
+
3
+ begin
4
+ gem "activejob", ">= 7.0"
5
+ require "active_job"
6
+
7
+ module Sidekiq
8
+ module ActiveJob
9
+ # @api private
10
+ class Wrapper
11
+ include Sidekiq::Job
12
+
13
+ def perform(job_data)
14
+ ::ActiveJob::Base.execute(job_data.merge("provider_job_id" => jid))
15
+ end
16
+ end
17
+ end
18
+ end
19
+
20
+ ActiveSupport.on_load(:active_job) do
21
+ # By including the Options module, we allow AJs to directly control sidekiq features
22
+ # via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
23
+ # AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
24
+ # manually retried, don't automatically die, etc.
25
+ #
26
+ # class SomeJob < ActiveJob::Base
27
+ # queue_as :default
28
+ # sidekiq_options retry: 3, backtrace: 10
29
+ # def perform
30
+ # end
31
+ # end
32
+ include Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
33
+ end
34
+
35
+ # Patch the ActiveJob module
36
+ module ActiveJob
37
+ module QueueAdapters
38
+ # Explicitly remove the implementation existing in older Rails.
39
+ remove_const(:SidekiqAdapter) if const_defined?(:SidekiqAdapter)
40
+
41
+ # Sidekiq adapter for Active Job
42
+ #
43
+ # To use Sidekiq set the queue_adapter config to +:sidekiq+.
44
+ #
45
+ # Rails.application.config.active_job.queue_adapter = :sidekiq
46
+ class SidekiqAdapter < AbstractAdapter
47
+ @@stopping = false
48
+
49
+ callback = -> { @@stopping = true }
50
+
51
+ Sidekiq.configure_client { |config| config.on(:quiet, &callback) }
52
+ Sidekiq.configure_server { |config| config.on(:quiet, &callback) }
53
+
54
+ # Defines whether enqueuing should happen implicitly to after commit when called
55
+ # from inside a transaction.
56
+ # @api private
57
+ def enqueue_after_transaction_commit?
58
+ true
59
+ end
60
+
61
+ # @api private
62
+ def enqueue(job)
63
+ job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(
64
+ wrapped: job.class,
65
+ queue: job.queue_name
66
+ ).perform_async(job.serialize)
67
+ end
68
+
69
+ # @api private
70
+ def enqueue_at(job, timestamp)
71
+ job.provider_job_id = Sidekiq::ActiveJob::Wrapper.set(
72
+ wrapped: job.class,
73
+ queue: job.queue_name
74
+ ).perform_at(timestamp, job.serialize)
75
+ end
76
+
77
+ # @api private
78
+ def enqueue_all(jobs)
79
+ enqueued_count = 0
80
+ jobs.group_by(&:class).each do |job_class, same_class_jobs|
81
+ same_class_jobs.group_by(&:queue_name).each do |queue, same_class_and_queue_jobs|
82
+ immediate_jobs, scheduled_jobs = same_class_and_queue_jobs.partition { |job| job.scheduled_at.nil? }
83
+
84
+ if immediate_jobs.any?
85
+ jids = Sidekiq::Client.push_bulk(
86
+ "class" => Sidekiq::ActiveJob::Wrapper,
87
+ "wrapped" => job_class,
88
+ "queue" => queue,
89
+ "args" => immediate_jobs.map { |job| [job.serialize] }
90
+ )
91
+ enqueued_count += jids.compact.size
92
+ end
93
+
94
+ if scheduled_jobs.any?
95
+ jids = Sidekiq::Client.push_bulk(
96
+ "class" => Sidekiq::ActiveJob::Wrapper,
97
+ "wrapped" => job_class,
98
+ "queue" => queue,
99
+ "args" => scheduled_jobs.map { |job| [job.serialize] },
100
+ "at" => scheduled_jobs.map { |job| job.scheduled_at&.to_f }
101
+ )
102
+ enqueued_count += jids.compact.size
103
+ end
104
+ end
105
+ end
106
+ enqueued_count
107
+ end
108
+
109
+ # @api private
110
+ def stopping? = !!@@stopping
111
+
112
+ # Defines a class alias for backwards compatibility with enqueued Active Job jobs.
113
+ # @api private
114
+ JobWrapper = Sidekiq::ActiveJob::Wrapper
115
+ end
116
+ end
117
+ end
118
+ rescue Gem::LoadError
119
+ # ActiveJob not available or version requirement not met
120
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require "rails/generators/named_base"
2
4
 
3
5
  module Sidekiq