sidekiq 5.2.7 → 8.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (169) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +845 -8
  3. data/LICENSE.txt +9 -0
  4. data/README.md +54 -54
  5. data/bin/multi_queue_bench +271 -0
  6. data/bin/sidekiq +22 -3
  7. data/bin/sidekiqload +219 -112
  8. data/bin/sidekiqmon +11 -0
  9. data/bin/webload +69 -0
  10. data/lib/active_job/queue_adapters/sidekiq_adapter.rb +120 -0
  11. data/lib/generators/sidekiq/job_generator.rb +59 -0
  12. data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
  13. data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
  14. data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
  15. data/lib/sidekiq/api.rb +757 -373
  16. data/lib/sidekiq/capsule.rb +132 -0
  17. data/lib/sidekiq/cli.rb +210 -233
  18. data/lib/sidekiq/client.rb +145 -103
  19. data/lib/sidekiq/component.rb +128 -0
  20. data/lib/sidekiq/config.rb +315 -0
  21. data/lib/sidekiq/deploy.rb +64 -0
  22. data/lib/sidekiq/embedded.rb +64 -0
  23. data/lib/sidekiq/fetch.rb +49 -42
  24. data/lib/sidekiq/iterable_job.rb +56 -0
  25. data/lib/sidekiq/job/interrupt_handler.rb +24 -0
  26. data/lib/sidekiq/job/iterable/active_record_enumerator.rb +53 -0
  27. data/lib/sidekiq/job/iterable/csv_enumerator.rb +47 -0
  28. data/lib/sidekiq/job/iterable/enumerators.rb +135 -0
  29. data/lib/sidekiq/job/iterable.rb +306 -0
  30. data/lib/sidekiq/job.rb +385 -0
  31. data/lib/sidekiq/job_logger.rb +34 -7
  32. data/lib/sidekiq/job_retry.rb +164 -109
  33. data/lib/sidekiq/job_util.rb +113 -0
  34. data/lib/sidekiq/launcher.rb +208 -107
  35. data/lib/sidekiq/logger.rb +80 -0
  36. data/lib/sidekiq/manager.rb +42 -46
  37. data/lib/sidekiq/metrics/query.rb +184 -0
  38. data/lib/sidekiq/metrics/shared.rb +109 -0
  39. data/lib/sidekiq/metrics/tracking.rb +150 -0
  40. data/lib/sidekiq/middleware/chain.rb +113 -56
  41. data/lib/sidekiq/middleware/current_attributes.rb +119 -0
  42. data/lib/sidekiq/middleware/i18n.rb +7 -7
  43. data/lib/sidekiq/middleware/modules.rb +23 -0
  44. data/lib/sidekiq/monitor.rb +147 -0
  45. data/lib/sidekiq/paginator.rb +41 -16
  46. data/lib/sidekiq/processor.rb +146 -127
  47. data/lib/sidekiq/profiler.rb +72 -0
  48. data/lib/sidekiq/rails.rb +46 -43
  49. data/lib/sidekiq/redis_client_adapter.rb +113 -0
  50. data/lib/sidekiq/redis_connection.rb +79 -108
  51. data/lib/sidekiq/ring_buffer.rb +31 -0
  52. data/lib/sidekiq/scheduled.rb +112 -50
  53. data/lib/sidekiq/sd_notify.rb +149 -0
  54. data/lib/sidekiq/systemd.rb +26 -0
  55. data/lib/sidekiq/testing/inline.rb +6 -5
  56. data/lib/sidekiq/testing.rb +91 -90
  57. data/lib/sidekiq/transaction_aware_client.rb +51 -0
  58. data/lib/sidekiq/version.rb +7 -1
  59. data/lib/sidekiq/web/action.rb +125 -60
  60. data/lib/sidekiq/web/application.rb +363 -259
  61. data/lib/sidekiq/web/config.rb +120 -0
  62. data/lib/sidekiq/web/csrf_protection.rb +183 -0
  63. data/lib/sidekiq/web/helpers.rb +241 -120
  64. data/lib/sidekiq/web/router.rb +62 -71
  65. data/lib/sidekiq/web.rb +69 -161
  66. data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
  67. data/lib/sidekiq.rb +94 -182
  68. data/sidekiq.gemspec +26 -16
  69. data/web/assets/images/apple-touch-icon.png +0 -0
  70. data/web/assets/javascripts/application.js +150 -61
  71. data/web/assets/javascripts/base-charts.js +120 -0
  72. data/web/assets/javascripts/chart.min.js +13 -0
  73. data/web/assets/javascripts/chartjs-adapter-date-fns.min.js +7 -0
  74. data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
  75. data/web/assets/javascripts/dashboard-charts.js +194 -0
  76. data/web/assets/javascripts/dashboard.js +41 -293
  77. data/web/assets/javascripts/metrics.js +280 -0
  78. data/web/assets/stylesheets/style.css +766 -0
  79. data/web/locales/ar.yml +72 -65
  80. data/web/locales/cs.yml +63 -62
  81. data/web/locales/da.yml +61 -53
  82. data/web/locales/de.yml +66 -53
  83. data/web/locales/el.yml +44 -24
  84. data/web/locales/en.yml +94 -66
  85. data/web/locales/es.yml +92 -54
  86. data/web/locales/fa.yml +66 -65
  87. data/web/locales/fr.yml +83 -62
  88. data/web/locales/gd.yml +99 -0
  89. data/web/locales/he.yml +66 -64
  90. data/web/locales/hi.yml +60 -59
  91. data/web/locales/it.yml +93 -54
  92. data/web/locales/ja.yml +75 -64
  93. data/web/locales/ko.yml +53 -52
  94. data/web/locales/lt.yml +84 -0
  95. data/web/locales/nb.yml +62 -61
  96. data/web/locales/nl.yml +53 -52
  97. data/web/locales/pl.yml +46 -45
  98. data/web/locales/{pt-br.yml → pt-BR.yml} +84 -56
  99. data/web/locales/pt.yml +52 -51
  100. data/web/locales/ru.yml +69 -63
  101. data/web/locales/sv.yml +54 -53
  102. data/web/locales/ta.yml +61 -60
  103. data/web/locales/tr.yml +101 -0
  104. data/web/locales/uk.yml +86 -61
  105. data/web/locales/ur.yml +65 -64
  106. data/web/locales/vi.yml +84 -0
  107. data/web/locales/zh-CN.yml +106 -0
  108. data/web/locales/{zh-tw.yml → zh-TW.yml} +43 -9
  109. data/web/views/_footer.erb +31 -19
  110. data/web/views/_job_info.erb +94 -75
  111. data/web/views/_metrics_period_select.erb +15 -0
  112. data/web/views/_nav.erb +14 -21
  113. data/web/views/_paging.erb +23 -19
  114. data/web/views/_poll_link.erb +3 -6
  115. data/web/views/_summary.erb +23 -23
  116. data/web/views/busy.erb +139 -87
  117. data/web/views/dashboard.erb +82 -53
  118. data/web/views/dead.erb +31 -27
  119. data/web/views/filtering.erb +6 -0
  120. data/web/views/layout.erb +15 -29
  121. data/web/views/metrics.erb +84 -0
  122. data/web/views/metrics_for_job.erb +58 -0
  123. data/web/views/morgue.erb +60 -70
  124. data/web/views/profiles.erb +43 -0
  125. data/web/views/queue.erb +50 -39
  126. data/web/views/queues.erb +45 -29
  127. data/web/views/retries.erb +65 -75
  128. data/web/views/retry.erb +32 -27
  129. data/web/views/scheduled.erb +58 -52
  130. data/web/views/scheduled_job_info.erb +1 -1
  131. metadata +96 -76
  132. data/.circleci/config.yml +0 -61
  133. data/.github/contributing.md +0 -32
  134. data/.github/issue_template.md +0 -11
  135. data/.gitignore +0 -15
  136. data/.travis.yml +0 -11
  137. data/3.0-Upgrade.md +0 -70
  138. data/4.0-Upgrade.md +0 -53
  139. data/5.0-Upgrade.md +0 -56
  140. data/COMM-LICENSE +0 -97
  141. data/Ent-Changes.md +0 -238
  142. data/Gemfile +0 -23
  143. data/LICENSE +0 -9
  144. data/Pro-2.0-Upgrade.md +0 -138
  145. data/Pro-3.0-Upgrade.md +0 -44
  146. data/Pro-4.0-Upgrade.md +0 -35
  147. data/Pro-Changes.md +0 -759
  148. data/Rakefile +0 -9
  149. data/bin/sidekiqctl +0 -20
  150. data/code_of_conduct.md +0 -50
  151. data/lib/generators/sidekiq/worker_generator.rb +0 -49
  152. data/lib/sidekiq/core_ext.rb +0 -1
  153. data/lib/sidekiq/ctl.rb +0 -221
  154. data/lib/sidekiq/delay.rb +0 -42
  155. data/lib/sidekiq/exception_handler.rb +0 -29
  156. data/lib/sidekiq/extensions/action_mailer.rb +0 -57
  157. data/lib/sidekiq/extensions/active_record.rb +0 -40
  158. data/lib/sidekiq/extensions/class_methods.rb +0 -40
  159. data/lib/sidekiq/extensions/generic_proxy.rb +0 -31
  160. data/lib/sidekiq/logging.rb +0 -122
  161. data/lib/sidekiq/middleware/server/active_record.rb +0 -23
  162. data/lib/sidekiq/util.rb +0 -66
  163. data/lib/sidekiq/worker.rb +0 -220
  164. data/web/assets/stylesheets/application-rtl.css +0 -246
  165. data/web/assets/stylesheets/application.css +0 -1144
  166. data/web/assets/stylesheets/bootstrap-rtl.min.css +0 -9
  167. data/web/assets/stylesheets/bootstrap.css +0 -5
  168. data/web/locales/zh-cn.yml +0 -68
  169. data/web/views/_status.erb +0 -4
@@ -0,0 +1,120 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "sidekiq/web/csrf_protection"
4
+
5
+ module Sidekiq
6
+ class Web
7
+ ##
8
+ # Configure the Sidekiq::Web instance in this process:
9
+ #
10
+ # require "sidekiq/web"
11
+ # Sidekiq::Web.configure do |config|
12
+ # config.register(MyExtension, name: "myext", tab: "TabName", index: "tabpage/")
13
+ # end
14
+ #
15
+ # This should go in your `config/routes.rb` or similar. It
16
+ # does not belong in your initializer since Web should not be
17
+ # loaded in some processes (like an actual Sidekiq process).
18
+ # See `examples/webui-ext` for a sample web extension.
19
+ class Config
20
+ extend Forwardable
21
+
22
+ OPTIONS = {
23
+ # By default we support direct uploads to p.f.c since the UI is a JS SPA
24
+ # and very difficult for us to vendor or provide ourselves. If you are worried
25
+ # about data security and wish to self-host, you can change these URLs.
26
+ profile_view_url: "https://profiler.firefox.com/public/%s",
27
+ profile_store_url: "https://api.profiler.firefox.com/compressed-store",
28
+ # Will be false in Sidekiq 9.0.
29
+ # CSRF is unnecessary if you are using SameSite=(Strict|Lax) cookies.
30
+ csrf: true
31
+ }
32
+
33
+ ##
34
+ # Allows users to add custom rows to all of the Job
35
+ # tables, e.g. Retries, Dead, Scheduled, with custom
36
+ # links to other systems, see _job_info.erb and test
37
+ # in web_test.rb
38
+ #
39
+ # Sidekiq::Web.configure do |cfg|
40
+ # cfg.custom_job_info_rows << JobLogLink.new
41
+ # end
42
+ #
43
+ # class JobLogLink
44
+ # def add_pair(job)
45
+ # yield "External Logs", "<a href='https://example.com/logs/#{job.jid}'>Logs for #{job.jid}</a>"
46
+ # end
47
+ # end
48
+ attr_accessor :custom_job_info_rows
49
+
50
+ attr_reader :tabs
51
+ attr_reader :locales
52
+ attr_reader :views
53
+ attr_reader :middlewares
54
+
55
+ # Adds the "Back to App" link in the header
56
+ attr_accessor :app_url
57
+
58
+ def initialize
59
+ @options = OPTIONS.dup
60
+ @locales = LOCALES
61
+ @views = VIEWS
62
+ @tabs = DEFAULT_TABS.dup
63
+ @middlewares = []
64
+ @custom_job_info_rows = []
65
+ end
66
+
67
+ def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
68
+
69
+ def use(*args, &block)
70
+ middlewares << [args, block]
71
+ end
72
+
73
+ # Register a class as a Sidekiq Web UI extension. The class should
74
+ # provide one or more tabs which map to an index route. Options:
75
+ #
76
+ # @param extclass [Class] Class which contains the HTTP actions, required
77
+ # @param name [String] the name of the extension, used to namespace assets
78
+ # @param tab [String | Array] labels(s) of the UI tabs
79
+ # @param index [String | Array] index route(s) for each tab
80
+ # @param root_dir [String] directory location to find assets, locales and views, typically `web/` within the gemfile
81
+ # @param asset_paths [Array] one or more directories under {root}/assets/{name} to be publicly served, e.g. ["js", "css", "img"]
82
+ # @param cache_for [Integer] amount of time to cache assets, default one day
83
+ #
84
+ # Web extensions will have a root `web/` directory with `locales/`, `assets/`
85
+ # and `views/` subdirectories.
86
+ def register_extension(extclass, name:, tab:, index:, root_dir: nil, cache_for: 86400, asset_paths: nil)
87
+ tab = Array(tab)
88
+ index = Array(index)
89
+ tab.zip(index).each do |tab, index|
90
+ tabs[tab] = index
91
+ end
92
+ if root_dir
93
+ locdir = File.join(root_dir, "locales")
94
+ locales << locdir if File.directory?(locdir)
95
+
96
+ if asset_paths && name
97
+ # if you have {root}/assets/{name}/js/scripts.js
98
+ # and {root}/assets/{name}/css/styles.css
99
+ # you would pass in:
100
+ # asset_paths: ["js", "css"]
101
+ # See script_tag and style_tag in web/helpers.rb
102
+ assdir = File.join(root_dir, "assets")
103
+ assurls = Array(asset_paths).map { |x| "/#{name}/#{x}" }
104
+ assetprops = {
105
+ urls: assurls,
106
+ root: assdir,
107
+ cascade: true
108
+ }
109
+ assetprops[:header_rules] = [[:all, {"cache-control" => "private, max-age=#{cache_for.to_i}"}]] if cache_for
110
+ middlewares << [[Rack::Static, assetprops], nil]
111
+ end
112
+ end
113
+
114
+ yield self if block_given?
115
+ extclass.registered(Web::Application)
116
+ end
117
+ alias_method :register, :register_extension
118
+ end
119
+ end
120
+ end
@@ -0,0 +1,183 @@
1
+ # frozen_string_literal: true
2
+
3
+ # this file originally based on authenticity_token.rb from the sinatra/rack-protection project
4
+ #
5
+ # The MIT License (MIT)
6
+ #
7
+ # Copyright (c) 2011-2017 Konstantin Haase
8
+ # Copyright (c) 2015-2017 Zachary Scott
9
+ #
10
+ # Permission is hereby granted, free of charge, to any person obtaining
11
+ # a copy of this software and associated documentation files (the
12
+ # 'Software'), to deal in the Software without restriction, including
13
+ # without limitation the rights to use, copy, modify, merge, publish,
14
+ # distribute, sublicense, and/or sell copies of the Software, and to
15
+ # permit persons to whom the Software is furnished to do so, subject to
16
+ # the following conditions:
17
+ #
18
+ # The above copyright notice and this permission notice shall be
19
+ # included in all copies or substantial portions of the Software.
20
+ #
21
+ # THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
22
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
23
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
24
+ # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
25
+ # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
26
+ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
27
+ # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
28
+
29
+ require "securerandom"
30
+ require "rack/request"
31
+
32
+ module Sidekiq
33
+ class Web
34
+ class CsrfProtection
35
+ def initialize(app, options = nil)
36
+ @app = app
37
+ end
38
+
39
+ def call(env)
40
+ accept?(env) ? admit(env) : deny(env)
41
+ end
42
+
43
+ private
44
+
45
+ def admit(env)
46
+ # On each successful request, we create a fresh masked token
47
+ # which will be used in any forms rendered for this request.
48
+ s = session(env)
49
+ s[:csrf] ||= SecureRandom.base64(TOKEN_LENGTH)
50
+ env[:csrf_token] = mask_token(s[:csrf])
51
+ @app.call(env)
52
+ end
53
+
54
+ def safe?(env)
55
+ %w[GET HEAD OPTIONS TRACE].include? env["REQUEST_METHOD"]
56
+ end
57
+
58
+ def logger(env)
59
+ @logger ||= env["rack.logger"] || ::Logger.new(env["rack.errors"])
60
+ end
61
+
62
+ def deny(env)
63
+ logger(env).warn "attack prevented by #{self.class}"
64
+ [403, {Rack::CONTENT_TYPE => "text/plain"}, ["Forbidden"]]
65
+ end
66
+
67
+ def session(env)
68
+ env["rack.session"] || fail(<<~EOM)
69
+ Sidekiq::Web needs a valid Rack session for CSRF protection. If this is a Rails app,
70
+ make sure you mount Sidekiq::Web *inside* your application routes:
71
+
72
+
73
+ Rails.application.routes.draw do
74
+ mount Sidekiq::Web => "/sidekiq"
75
+ ....
76
+ end
77
+
78
+
79
+ If this is a Rails app in API mode, you need to enable sessions.
80
+
81
+ https://guides.rubyonrails.org/api_app.html#using-session-middlewares
82
+
83
+ If this is a bare Rack app, use a session middleware before Sidekiq::Web:
84
+
85
+ # first, use IRB to create a shared secret key for sessions and commit it
86
+ require 'securerandom'; File.open(".session.key", "w") {|f| f.write(SecureRandom.hex(32)) }
87
+
88
+ # now use the secret with a session cookie middleware
89
+ use Rack::Session::Cookie, secret: File.read(".session.key"), same_site: true, max_age: 86400
90
+ run Sidekiq::Web
91
+
92
+ EOM
93
+ end
94
+
95
+ def accept?(env)
96
+ return true if safe?(env)
97
+
98
+ giventoken = ::Rack::Request.new(env).params["authenticity_token"]
99
+ valid_token?(env, giventoken)
100
+ end
101
+
102
+ TOKEN_LENGTH = 32
103
+
104
+ # Checks that the token given to us as a parameter matches
105
+ # the token stored in the session.
106
+ def valid_token?(env, giventoken)
107
+ return false if giventoken.nil? || giventoken.empty?
108
+
109
+ begin
110
+ token = decode_token(giventoken)
111
+ rescue ArgumentError # client input is invalid
112
+ return false
113
+ end
114
+
115
+ sess = session(env)
116
+ localtoken = sess[:csrf]
117
+
118
+ # Checks that Rack::Session::Cookie actually contains the csrf token
119
+ return false if localtoken.nil?
120
+
121
+ # Rotate the session token after every use
122
+ sess[:csrf] = SecureRandom.base64(TOKEN_LENGTH)
123
+
124
+ # See if it's actually a masked token or not. We should be able
125
+ # to handle any unmasked tokens that we've issued without error.
126
+
127
+ if unmasked_token?(token)
128
+ compare_with_real_token token, localtoken
129
+ elsif masked_token?(token)
130
+ unmasked = unmask_token(token)
131
+ compare_with_real_token unmasked, localtoken
132
+ else
133
+ false # Token is malformed
134
+ end
135
+ end
136
+
137
+ # Creates a masked version of the authenticity token that varies
138
+ # on each request. The masking is used to mitigate SSL attacks
139
+ # like BREACH.
140
+ def mask_token(token)
141
+ token = decode_token(token)
142
+ one_time_pad = SecureRandom.random_bytes(token.length)
143
+ encrypted_token = xor_byte_strings(one_time_pad, token)
144
+ masked_token = one_time_pad + encrypted_token
145
+ encode_token(masked_token)
146
+ end
147
+
148
+ # Essentially the inverse of +mask_token+.
149
+ def unmask_token(masked_token)
150
+ # Split the token into the one-time pad and the encrypted
151
+ # value and decrypt it
152
+ token_length = masked_token.length / 2
153
+ one_time_pad = masked_token[0...token_length]
154
+ encrypted_token = masked_token[token_length..]
155
+ xor_byte_strings(one_time_pad, encrypted_token)
156
+ end
157
+
158
+ def unmasked_token?(token)
159
+ token.length == TOKEN_LENGTH
160
+ end
161
+
162
+ def masked_token?(token)
163
+ token.length == TOKEN_LENGTH * 2
164
+ end
165
+
166
+ def compare_with_real_token(token, local)
167
+ ::Rack::Utils.secure_compare(token.to_s, decode_token(local).to_s)
168
+ end
169
+
170
+ def encode_token(token)
171
+ [token].pack("m0").tr("+/", "-_")
172
+ end
173
+
174
+ def decode_token(token)
175
+ token.tr("-_", "+/").unpack1("m0")
176
+ end
177
+
178
+ def xor_byte_strings(s1, s2)
179
+ s1.bytes.zip(s2.bytes).map { |(c1, c2)| c1 ^ c2 }.pack("c*")
180
+ end
181
+ end
182
+ end
183
+ end