sidekiq 6.4.1 → 6.5.10

Sign up to get free protection for your applications and to get access to all the features.
Files changed (67) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +107 -1
  3. data/README.md +1 -1
  4. data/bin/sidekiqload +18 -12
  5. data/lib/sidekiq/api.rb +222 -71
  6. data/lib/sidekiq/cli.rb +51 -37
  7. data/lib/sidekiq/client.rb +27 -28
  8. data/lib/sidekiq/component.rb +65 -0
  9. data/lib/sidekiq/delay.rb +1 -1
  10. data/lib/sidekiq/extensions/generic_proxy.rb +1 -1
  11. data/lib/sidekiq/fetch.rb +18 -16
  12. data/lib/sidekiq/job_retry.rb +73 -52
  13. data/lib/sidekiq/job_util.rb +15 -9
  14. data/lib/sidekiq/launcher.rb +37 -33
  15. data/lib/sidekiq/logger.rb +5 -19
  16. data/lib/sidekiq/manager.rb +28 -25
  17. data/lib/sidekiq/metrics/deploy.rb +47 -0
  18. data/lib/sidekiq/metrics/query.rb +153 -0
  19. data/lib/sidekiq/metrics/shared.rb +94 -0
  20. data/lib/sidekiq/metrics/tracking.rb +134 -0
  21. data/lib/sidekiq/middleware/chain.rb +82 -38
  22. data/lib/sidekiq/middleware/current_attributes.rb +18 -12
  23. data/lib/sidekiq/middleware/i18n.rb +6 -4
  24. data/lib/sidekiq/middleware/modules.rb +21 -0
  25. data/lib/sidekiq/monitor.rb +2 -2
  26. data/lib/sidekiq/paginator.rb +11 -3
  27. data/lib/sidekiq/processor.rb +47 -41
  28. data/lib/sidekiq/rails.rb +25 -8
  29. data/lib/sidekiq/redis_client_adapter.rb +154 -0
  30. data/lib/sidekiq/redis_connection.rb +80 -49
  31. data/lib/sidekiq/ring_buffer.rb +29 -0
  32. data/lib/sidekiq/scheduled.rb +53 -24
  33. data/lib/sidekiq/testing/inline.rb +4 -4
  34. data/lib/sidekiq/testing.rb +37 -36
  35. data/lib/sidekiq/transaction_aware_client.rb +45 -0
  36. data/lib/sidekiq/version.rb +1 -1
  37. data/lib/sidekiq/web/action.rb +3 -3
  38. data/lib/sidekiq/web/application.rb +21 -5
  39. data/lib/sidekiq/web/csrf_protection.rb +2 -2
  40. data/lib/sidekiq/web/helpers.rb +20 -7
  41. data/lib/sidekiq/web.rb +5 -1
  42. data/lib/sidekiq/worker.rb +24 -16
  43. data/lib/sidekiq.rb +106 -31
  44. data/sidekiq.gemspec +2 -2
  45. data/web/assets/javascripts/application.js +59 -26
  46. data/web/assets/javascripts/chart.min.js +13 -0
  47. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  48. data/web/assets/javascripts/dashboard.js +0 -17
  49. data/web/assets/javascripts/graph.js +16 -0
  50. data/web/assets/javascripts/metrics.js +262 -0
  51. data/web/assets/stylesheets/application.css +45 -3
  52. data/web/locales/el.yml +43 -19
  53. data/web/locales/en.yml +7 -0
  54. data/web/locales/ja.yml +7 -0
  55. data/web/locales/pt-br.yml +27 -9
  56. data/web/locales/zh-cn.yml +36 -11
  57. data/web/locales/zh-tw.yml +32 -7
  58. data/web/views/_nav.erb +1 -1
  59. data/web/views/_summary.erb +1 -1
  60. data/web/views/busy.erb +9 -4
  61. data/web/views/dashboard.erb +1 -0
  62. data/web/views/metrics.erb +69 -0
  63. data/web/views/metrics_for_job.erb +87 -0
  64. data/web/views/queue.erb +5 -1
  65. metadata +34 -9
  66. data/lib/sidekiq/exception_handler.rb +0 -27
  67. data/lib/sidekiq/util.rb +0 -108
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 99c9e264c092b88ea726be158fafe5bbab91f82f4b5864dee406280622e98e4b
4
- data.tar.gz: acd72bd99929d7c9d129cb9662276cc5adb7214de07cd4fc8accf6b9d521994a
3
+ metadata.gz: b12bc33857058ca07b776ff3f9e39eb98a84533c8f1c78367626a6a514d53607
4
+ data.tar.gz: d783f3060fc348786c7128ffc3b14703d9ee676a1a4972d88bc2e995fbabfd16
5
5
  SHA512:
6
- metadata.gz: 622c25276c017302c1a9d144e9366043ba359b2c3b0c57d4e7baad8f9de2e9c9969a86c91acdbefcf736af92e297c4e1fbe2008aa41e0c1accadda77dd0724f5
7
- data.tar.gz: 7e64012a5368cb0158ecaa50cdea6447709a64dd3a2816b36a31e7f17d70fffff81bd8d317c0cc1f9a6317adcffad9c200c48f9ca4bf208afba819ff7a07738e
6
+ metadata.gz: 0471f10240cbf2a1e2337e50111690a32153da3fa23f3e8462ff545dcc9cde06d67574cc26ed1c2be3fed88d31a34002ec30641a6c40a4f668a44005e9d5d297
7
+ data.tar.gz: 88f36a02144dbdf6fa58eb5d58f9615d99321049a76717dd2a4bd03a0bc7ebfe69e4709b860afd22782c80e5a868d8c85026d15ce42824675e3f6f09534c6335
data/Changes.md CHANGED
@@ -2,7 +2,106 @@
2
2
 
3
3
  [Sidekiq Changes](https://github.com/mperham/sidekiq/blob/main/Changes.md) | [Sidekiq Pro Changes](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md) | [Sidekiq Enterprise Changes](https://github.com/mperham/sidekiq/blob/main/Ent-Changes.md)
4
4
 
5
- HEAD
5
+ 6.5.10
6
+ ----------
7
+
8
+ - Web UI DoS vector [#6045] CVE-2023-26141
9
+ - Fix broadcast logger with Rails 7.1 [#6054]
10
+
11
+ 6.5.9
12
+ ----------
13
+
14
+ - Ensure Sidekiq.options[:environment] == RAILS_ENV [#5932]
15
+
16
+ 6.5.8
17
+ ----------
18
+
19
+ - Fail if using a bad version of scout_apm [#5616]
20
+ - Add pagination to Busy page [#5556]
21
+ - Speed up WorkSet#each [#5559]
22
+ - Adjust CurrentAttributes to work with the String class name so we aren't referencing
23
+ the Class within a Rails initializer [#5536]
24
+
25
+ 6.5.7
26
+ ----------
27
+
28
+ - Updates for JA and ZH locales
29
+ - Further optimizations for scheduled polling [#5513]
30
+
31
+ 6.5.6
32
+ ----------
33
+
34
+ - Fix deprecation warnings with redis-rb 4.8.0 [#5484]
35
+ - Lock redis-rb to < 5.0 as we are moving to redis-client in Sidekiq 7.0
36
+
37
+ 6.5.5
38
+ ----------
39
+
40
+ - Fix require issue with job_retry.rb [#5462]
41
+ - Improve Sidekiq::Web compatibility with Rack 3.x
42
+
43
+ 6.5.4
44
+ ----------
45
+
46
+ - Fix invalid code on Ruby 2.5 [#5460]
47
+ - Fix further metrics dependency issues [#5457]
48
+
49
+ 6.5.3
50
+ ----------
51
+
52
+ - Don't require metrics code without explicit opt-in [#5456]
53
+
54
+ 6.5.2
55
+ ----------
56
+
57
+ - [Job Metrics are under active development, help wanted!](https://github.com/mperham/sidekiq/wiki/Metrics#contributing) **BETA**
58
+ - Add `Context` column on queue page which shows any CurrentAttributes [#5450]
59
+ - `sidekiq_retry_in` may now return `:discard` or `:kill` to dynamically stop job retries [#5406]
60
+ - Smarter sorting of processes in /busy Web UI [#5398]
61
+ - Fix broken hamburger menu in mobile UI [#5428]
62
+ - Require redis-rb 4.5.0. Note that Sidekiq will break if you use the
63
+ [`Redis.exists_returns_integer = false`](https://github.com/redis/redis-rb/blob/master/CHANGELOG.md#450) flag. [#5394]
64
+
65
+ 6.5.1
66
+ ----------
67
+
68
+ - Fix `push_bulk` breakage [#5387]
69
+
70
+ 6.5.0
71
+ ---------
72
+
73
+ - Substantial refactoring of Sidekiq server internals, part of a larger effort
74
+ to reduce Sidekiq's internal usage of global methods and data, see [docs/global_to_local.md](docs/global_to_local.md) and [docs/middleware.md](docs/middleware.md).
75
+ - **Add beta support for the `redis-client` gem**. This will become the default Redis driver in Sidekiq 7.0. [#5298]
76
+ Read more: https://github.com/mperham/sidekiq/wiki/Using-redis-client
77
+ - **Add beta support for DB transaction-aware client** [#5291]
78
+ Add this line to your initializer and any jobs created during a transaction
79
+ will only be pushed to Redis **after the transaction commits**. You will need to add the
80
+ `after_commit_everywhere` gem to your Gemfile.
81
+ ```ruby
82
+ Sidekiq.transactional_push!
83
+ ```
84
+ This feature does not have a lot of production usage yet; please try it out and let us
85
+ know if you have any issues. It will be fully supported in Sidekiq 7.0 or removed if it
86
+ proves problematic.
87
+ - Fix regression with middleware arguments [#5312]
88
+
89
+ 6.4.2
90
+ ---------
91
+
92
+ - Strict argument checking now runs after client-side middleware [#5246]
93
+ - Fix page events with live polling [#5184]
94
+ - Many under-the-hood changes to remove all usage of the term "worker"
95
+ from the Sidekiq codebase and APIs. This mostly involved RDoc and local
96
+ variable names but a few constants and public APIs were changed. The old
97
+ APIs will be removed in Sidekiq 7.0.
98
+ ```
99
+ Sidekiq::DEFAULT_WORKER_OPTIONS -> Sidekiq.default_job_options
100
+ Sidekiq.default_worker_options -> Sidekiq.default_job_options
101
+ Sidekiq::Queues["default"].jobs_by_worker(HardJob) -> Sidekiq::Queues["default"].jobs_by_class(HardJob)
102
+ ```
103
+
104
+ 6.4.1
6
105
  ---------
7
106
 
8
107
  - Fix pipeline/multi deprecations in redis-rb 4.6
@@ -319,6 +418,13 @@ See the [Logging wiki page](https://github.com/mperham/sidekiq/wiki/Logging) for
319
418
  - Integrate the StandardRB code formatter to ensure consistent code
320
419
  styling. [#4114, gearnode]
321
420
 
421
+ 5.2.10
422
+ ---------
423
+
424
+ - Backport fix for CVE-2022-23837.
425
+ - Migrate to `exists?` for redis-rb.
426
+ - Lock redis-rb to <4.6 to avoid deprecations.
427
+
322
428
  5.2.9
323
429
  ---------
324
430
 
data/README.md CHANGED
@@ -36,7 +36,7 @@ Sidekiq 6.0 supports Rails 5.0+ but does not require it.
36
36
  Installation
37
37
  -----------------
38
38
 
39
- gem install sidekiq
39
+ bundle add sidekiq
40
40
 
41
41
 
42
42
  Getting Started
data/bin/sidekiqload CHANGED
@@ -11,6 +11,10 @@ Bundler.require(:default, :load_test)
11
11
  require_relative "../lib/sidekiq/cli"
12
12
  require_relative "../lib/sidekiq/launcher"
13
13
 
14
+ if ENV["SIDEKIQ_REDIS_CLIENT"]
15
+ Sidekiq::RedisConnection.adapter = :redis_client
16
+ end
17
+
14
18
  Sidekiq.configure_server do |config|
15
19
  config.options[:concurrency] = 10
16
20
  config.redis = {db: 13, port: 6380}
@@ -36,7 +40,6 @@ end
36
40
 
37
41
  # brew tap shopify/shopify
38
42
  # brew install toxiproxy
39
- # gem install toxiproxy
40
43
  # run `toxiproxy-server` in a separate terminal window.
41
44
  require "toxiproxy"
42
45
  # simulate a non-localhost network for realer-world conditions.
@@ -86,16 +89,11 @@ def Process.rss
86
89
  `ps -o rss= -p #{Process.pid}`.chomp.to_i
87
90
  end
88
91
 
89
- iter = 50
92
+ iter = 10
90
93
  count = 10_000
91
94
 
92
95
  iter.times do
93
- arr = Array.new(count) do
94
- []
95
- end
96
- count.times do |idx|
97
- arr[idx][0] = idx
98
- end
96
+ arr = Array.new(count) { |idx| [idx] }
99
97
  Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
100
98
  end
101
99
  Sidekiq.logger.error "Created #{count * iter} jobs"
@@ -124,16 +122,24 @@ Monitoring = Thread.new do
124
122
  end
125
123
  end
126
124
 
125
+ def with_latency(latency, &block)
126
+ Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
127
+ if latency > 0
128
+ Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
129
+ else
130
+ yield
131
+ end
132
+ end
133
+
127
134
  begin
128
135
  # RubyProf::exclude_threads = [ Monitoring ]
129
136
  # RubyProf.start
130
137
  events = Sidekiq.options[:lifecycle_events][:startup]
131
138
  events.each(&:call)
132
139
  events.clear
133
-
134
- Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
135
- Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
136
- launcher = Sidekiq::Launcher.new(Sidekiq.options)
140
+
141
+ with_latency(Integer(ENV.fetch("LATENCY", "1"))) do
142
+ launcher = Sidekiq::Launcher.new(Sidekiq)
137
143
  launcher.run
138
144
 
139
145
  while readable_io = IO.select([self_read])