sidekiq 5.2.10 → 6.5.6
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +391 -1
- data/LICENSE +3 -3
- data/README.md +24 -35
- data/bin/sidekiq +27 -3
- data/bin/sidekiqload +79 -67
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +504 -307
- data/lib/sidekiq/cli.rb +190 -206
- data/lib/sidekiq/client.rb +77 -81
- data/lib/sidekiq/component.rb +65 -0
- data/lib/sidekiq/delay.rb +8 -7
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +7 -5
- data/lib/sidekiq/fetch.rb +50 -40
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +33 -7
- data/lib/sidekiq/job_retry.rb +126 -106
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +177 -83
- data/lib/sidekiq/logger.rb +156 -0
- data/lib/sidekiq/manager.rb +40 -41
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +102 -46
- data/lib/sidekiq/middleware/current_attributes.rb +63 -0
- data/lib/sidekiq/middleware/i18n.rb +7 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +20 -16
- data/lib/sidekiq/processor.rb +104 -97
- data/lib/sidekiq/rails.rb +47 -37
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +108 -77
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +64 -35
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +6 -5
- data/lib/sidekiq/testing.rb +68 -58
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +100 -77
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +134 -94
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +65 -105
- data/lib/sidekiq/worker.rb +253 -106
- data/lib/sidekiq.rb +170 -62
- data/sidekiq.gemspec +23 -16
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +112 -61
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +53 -89
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +88 -233
- data/web/locales/ar.yml +8 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +13 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +7 -1
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +56 -22
- data/web/views/dashboard.erb +23 -14
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +3 -1
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +23 -10
- data/web/views/queues.erb +10 -2
- data/web/views/retries.erb +11 -8
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +5 -2
- metadata +53 -64
- data/.circleci/config.yml +0 -61
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -15
- data/.travis.yml +0 -11
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/COMM-LICENSE +0 -97
- data/Ent-Changes.md +0 -238
- data/Gemfile +0 -19
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-Changes.md +0 -759
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -20
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/ctl.rb +0 -221
- data/lib/sidekiq/exception_handler.rb +0 -29
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
- data/lib/sidekiq/util.rb +0 -66
data/lib/sidekiq/web.rb
CHANGED
@@ -1,20 +1,20 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require 'erb'
|
3
2
|
|
4
|
-
require
|
5
|
-
require 'sidekiq/api'
|
6
|
-
require 'sidekiq/paginator'
|
7
|
-
require 'sidekiq/web/helpers'
|
3
|
+
require "erb"
|
8
4
|
|
9
|
-
require
|
10
|
-
require
|
11
|
-
require
|
5
|
+
require "sidekiq"
|
6
|
+
require "sidekiq/api"
|
7
|
+
require "sidekiq/paginator"
|
8
|
+
require "sidekiq/web/helpers"
|
12
9
|
|
13
|
-
require
|
10
|
+
require "sidekiq/web/router"
|
11
|
+
require "sidekiq/web/action"
|
12
|
+
require "sidekiq/web/application"
|
13
|
+
require "sidekiq/web/csrf_protection"
|
14
14
|
|
15
|
-
require
|
16
|
-
require
|
17
|
-
require
|
15
|
+
require "rack/content_length"
|
16
|
+
require "rack/builder"
|
17
|
+
require "rack/static"
|
18
18
|
|
19
19
|
module Sidekiq
|
20
20
|
class Web
|
@@ -25,27 +25,23 @@ module Sidekiq
|
|
25
25
|
ASSETS = "#{ROOT}/assets"
|
26
26
|
|
27
27
|
DEFAULT_TABS = {
|
28
|
-
"Dashboard" =>
|
29
|
-
"Busy"
|
30
|
-
"Queues"
|
31
|
-
"Retries"
|
32
|
-
"Scheduled" =>
|
33
|
-
"Dead"
|
28
|
+
"Dashboard" => "",
|
29
|
+
"Busy" => "busy",
|
30
|
+
"Queues" => "queues",
|
31
|
+
"Retries" => "retries",
|
32
|
+
"Scheduled" => "scheduled",
|
33
|
+
"Dead" => "morgue"
|
34
34
|
}
|
35
35
|
|
36
|
+
if ENV["SIDEKIQ_METRICS_BETA"] == "1"
|
37
|
+
DEFAULT_TABS["Metrics"] = "metrics"
|
38
|
+
end
|
39
|
+
|
36
40
|
class << self
|
37
41
|
def settings
|
38
42
|
self
|
39
43
|
end
|
40
44
|
|
41
|
-
def middlewares
|
42
|
-
@middlewares ||= []
|
43
|
-
end
|
44
|
-
|
45
|
-
def use(*middleware_args, &block)
|
46
|
-
middlewares << [middleware_args, block]
|
47
|
-
end
|
48
|
-
|
49
45
|
def default_tabs
|
50
46
|
DEFAULT_TABS
|
51
47
|
end
|
@@ -64,39 +60,52 @@ module Sidekiq
|
|
64
60
|
end
|
65
61
|
|
66
62
|
def enable(*opts)
|
67
|
-
opts.each {|key| set(key, true) }
|
63
|
+
opts.each { |key| set(key, true) }
|
68
64
|
end
|
69
65
|
|
70
66
|
def disable(*opts)
|
71
|
-
opts.each {|key| set(key, false) }
|
67
|
+
opts.each { |key| set(key, false) }
|
68
|
+
end
|
69
|
+
|
70
|
+
def middlewares
|
71
|
+
@middlewares ||= []
|
72
|
+
end
|
73
|
+
|
74
|
+
def use(*args, &block)
|
75
|
+
middlewares << [args, block]
|
72
76
|
end
|
73
77
|
|
74
|
-
# Helper for the Sinatra syntax: Sidekiq::Web.set(:session_secret, Rails.application.secrets...)
|
75
78
|
def set(attribute, value)
|
76
79
|
send(:"#{attribute}=", value)
|
77
80
|
end
|
78
81
|
|
79
|
-
|
82
|
+
def sessions=(val)
|
83
|
+
puts "WARNING: Sidekiq::Web.sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
|
84
|
+
end
|
85
|
+
|
86
|
+
def session_secret=(val)
|
87
|
+
puts "WARNING: Sidekiq::Web.session_secret= is no longer relevant and will be removed in Sidekiq 7.0. #{caller(1..1).first}"
|
88
|
+
end
|
89
|
+
|
90
|
+
attr_accessor :app_url, :redis_pool
|
80
91
|
attr_writer :locales, :views
|
81
92
|
end
|
82
93
|
|
83
94
|
def self.inherited(child)
|
84
|
-
child.app_url =
|
85
|
-
child.
|
86
|
-
child.redis_pool = self.redis_pool
|
87
|
-
child.sessions = self.sessions
|
95
|
+
child.app_url = app_url
|
96
|
+
child.redis_pool = redis_pool
|
88
97
|
end
|
89
98
|
|
90
99
|
def settings
|
91
100
|
self.class.settings
|
92
101
|
end
|
93
102
|
|
94
|
-
def
|
95
|
-
middlewares
|
103
|
+
def middlewares
|
104
|
+
@middlewares ||= self.class.middlewares
|
96
105
|
end
|
97
106
|
|
98
|
-
def
|
99
|
-
|
107
|
+
def use(*args, &block)
|
108
|
+
middlewares << [args, block]
|
100
109
|
end
|
101
110
|
|
102
111
|
def call(env)
|
@@ -113,29 +122,19 @@ module Sidekiq
|
|
113
122
|
end
|
114
123
|
|
115
124
|
def enable(*opts)
|
116
|
-
opts.each {|key| set(key, true) }
|
125
|
+
opts.each { |key| set(key, true) }
|
117
126
|
end
|
118
127
|
|
119
128
|
def disable(*opts)
|
120
|
-
opts.each {|key| set(key, false) }
|
129
|
+
opts.each { |key| set(key, false) }
|
121
130
|
end
|
122
131
|
|
123
132
|
def set(attribute, value)
|
124
133
|
send(:"#{attribute}=", value)
|
125
134
|
end
|
126
135
|
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
attr_writer :sessions
|
131
|
-
|
132
|
-
def sessions
|
133
|
-
unless instance_variable_defined?("@sessions")
|
134
|
-
@sessions = self.class.sessions
|
135
|
-
@sessions = @sessions.to_hash.dup if @sessions.respond_to?(:to_hash)
|
136
|
-
end
|
137
|
-
|
138
|
-
@sessions
|
136
|
+
def sessions=(val)
|
137
|
+
puts "Sidekiq::Web#sessions= is no longer relevant and will be removed in Sidekiq 7.0. #{caller[2..2].first}"
|
139
138
|
end
|
140
139
|
|
141
140
|
def self.register(extension)
|
@@ -144,50 +143,20 @@ module Sidekiq
|
|
144
143
|
|
145
144
|
private
|
146
145
|
|
147
|
-
def using?(middleware)
|
148
|
-
middlewares.any? do |(m,_)|
|
149
|
-
m.kind_of?(Array) && (m[0] == middleware || m[0].kind_of?(middleware))
|
150
|
-
end
|
151
|
-
end
|
152
|
-
|
153
|
-
def build_sessions
|
154
|
-
middlewares = self.middlewares
|
155
|
-
|
156
|
-
unless using?(::Rack::Protection) || ENV['RACK_ENV'] == 'test'
|
157
|
-
middlewares.unshift [[::Rack::Protection, { use: :authenticity_token }], nil]
|
158
|
-
end
|
159
|
-
|
160
|
-
s = sessions
|
161
|
-
return unless s
|
162
|
-
|
163
|
-
unless using? ::Rack::Session::Cookie
|
164
|
-
unless secret = Web.session_secret
|
165
|
-
require 'securerandom'
|
166
|
-
secret = SecureRandom.hex(64)
|
167
|
-
end
|
168
|
-
|
169
|
-
options = { secret: secret }
|
170
|
-
options = options.merge(s.to_hash) if s.respond_to? :to_hash
|
171
|
-
|
172
|
-
middlewares.unshift [[::Rack::Session::Cookie, options], nil]
|
173
|
-
end
|
174
|
-
end
|
175
|
-
|
176
146
|
def build
|
177
|
-
build_sessions
|
178
|
-
|
179
|
-
middlewares = self.middlewares
|
180
147
|
klass = self.class
|
148
|
+
m = middlewares
|
181
149
|
|
182
|
-
|
183
|
-
|
184
|
-
map "/#{asset_dir}" do
|
185
|
-
run ::Rack::File.new("#{ASSETS}/#{asset_dir}", { 'Cache-Control' => 'public, max-age=86400' })
|
186
|
-
end
|
187
|
-
end
|
188
|
-
|
189
|
-
middlewares.each {|middleware, block| use(*middleware, &block) }
|
150
|
+
rules = []
|
151
|
+
rules = [[:all, {"cache-control" => "public, max-age=86400"}]] unless ENV["SIDEKIQ_WEB_TESTING"]
|
190
152
|
|
153
|
+
::Rack::Builder.new do
|
154
|
+
use Rack::Static, urls: ["/stylesheets", "/images", "/javascripts"],
|
155
|
+
root: ASSETS,
|
156
|
+
cascade: true,
|
157
|
+
header_rules: rules
|
158
|
+
m.each { |middleware, block| use(*middleware, &block) }
|
159
|
+
use Sidekiq::Web::CsrfProtection unless $TESTING
|
191
160
|
run WebApplication.new(klass)
|
192
161
|
end
|
193
162
|
end
|
@@ -196,18 +165,9 @@ module Sidekiq
|
|
196
165
|
Sidekiq::WebApplication.helpers WebHelpers
|
197
166
|
Sidekiq::WebApplication.helpers Sidekiq::Paginator
|
198
167
|
|
199
|
-
Sidekiq::WebAction.class_eval
|
200
|
-
|
201
|
-
|
202
|
-
if defined?(::ActionDispatch::Request::Session) &&
|
203
|
-
!::ActionDispatch::Request::Session.method_defined?(:each)
|
204
|
-
# mperham/sidekiq#2460
|
205
|
-
# Rack apps can't reuse the Rails session store without
|
206
|
-
# this monkeypatch, fixed in Rails 5.
|
207
|
-
class ActionDispatch::Request::Session
|
208
|
-
def each(&block)
|
209
|
-
hash = self.to_hash
|
210
|
-
hash.each(&block)
|
168
|
+
Sidekiq::WebAction.class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
169
|
+
def _render
|
170
|
+
#{ERB.new(File.read(Web::LAYOUT)).src}
|
211
171
|
end
|
212
|
-
|
172
|
+
RUBY
|
213
173
|
end
|
data/lib/sidekiq/worker.rb
CHANGED
@@ -1,14 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
require 'sidekiq/client'
|
3
2
|
|
4
|
-
|
3
|
+
require "sidekiq/client"
|
5
4
|
|
5
|
+
module Sidekiq
|
6
6
|
##
|
7
7
|
# Include this module in your worker class and you can easily create
|
8
8
|
# asynchronous jobs:
|
9
9
|
#
|
10
10
|
# class HardWorker
|
11
11
|
# include Sidekiq::Worker
|
12
|
+
# sidekiq_options queue: 'critical', retry: 5
|
12
13
|
#
|
13
14
|
# def perform(*args)
|
14
15
|
# # do some work
|
@@ -20,16 +21,145 @@ module Sidekiq
|
|
20
21
|
# HardWorker.perform_async(1, 2, 3)
|
21
22
|
#
|
22
23
|
# Note that perform_async is a class method, perform is an instance method.
|
24
|
+
#
|
25
|
+
# Sidekiq::Worker also includes several APIs to provide compatibility with
|
26
|
+
# ActiveJob.
|
27
|
+
#
|
28
|
+
# class SomeWorker
|
29
|
+
# include Sidekiq::Worker
|
30
|
+
# queue_as :critical
|
31
|
+
#
|
32
|
+
# def perform(...)
|
33
|
+
# end
|
34
|
+
# end
|
35
|
+
#
|
36
|
+
# SomeWorker.set(wait_until: 1.hour).perform_async(123)
|
37
|
+
#
|
38
|
+
# Note that arguments passed to the job must still obey Sidekiq's
|
39
|
+
# best practice for simple, JSON-native data types. Sidekiq will not
|
40
|
+
# implement ActiveJob's more complex argument serialization. For
|
41
|
+
# this reason, we don't implement `perform_later` as our call semantics
|
42
|
+
# are very different.
|
43
|
+
#
|
23
44
|
module Worker
|
45
|
+
##
|
46
|
+
# The Options module is extracted so we can include it in ActiveJob::Base
|
47
|
+
# and allow native AJs to configure Sidekiq features/internals.
|
48
|
+
module Options
|
49
|
+
def self.included(base)
|
50
|
+
base.extend(ClassMethods)
|
51
|
+
base.sidekiq_class_attribute :sidekiq_options_hash
|
52
|
+
base.sidekiq_class_attribute :sidekiq_retry_in_block
|
53
|
+
base.sidekiq_class_attribute :sidekiq_retries_exhausted_block
|
54
|
+
end
|
55
|
+
|
56
|
+
module ClassMethods
|
57
|
+
ACCESSOR_MUTEX = Mutex.new
|
58
|
+
|
59
|
+
##
|
60
|
+
# Allows customization for this type of Worker.
|
61
|
+
# Legal options:
|
62
|
+
#
|
63
|
+
# queue - name of queue to use for this job type, default *default*
|
64
|
+
# retry - enable retries for this Worker in case of error during execution,
|
65
|
+
# *true* to use the default or *Integer* count
|
66
|
+
# backtrace - whether to save any error backtrace in the retry payload to display in web UI,
|
67
|
+
# can be true, false or an integer number of lines to save, default *false*
|
68
|
+
#
|
69
|
+
# In practice, any option is allowed. This is the main mechanism to configure the
|
70
|
+
# options for a specific job.
|
71
|
+
def sidekiq_options(opts = {})
|
72
|
+
opts = opts.transform_keys(&:to_s) # stringify
|
73
|
+
self.sidekiq_options_hash = get_sidekiq_options.merge(opts)
|
74
|
+
end
|
75
|
+
|
76
|
+
def sidekiq_retry_in(&block)
|
77
|
+
self.sidekiq_retry_in_block = block
|
78
|
+
end
|
79
|
+
|
80
|
+
def sidekiq_retries_exhausted(&block)
|
81
|
+
self.sidekiq_retries_exhausted_block = block
|
82
|
+
end
|
83
|
+
|
84
|
+
def get_sidekiq_options # :nodoc:
|
85
|
+
self.sidekiq_options_hash ||= Sidekiq.default_job_options
|
86
|
+
end
|
87
|
+
|
88
|
+
def sidekiq_class_attribute(*attrs)
|
89
|
+
instance_reader = true
|
90
|
+
instance_writer = true
|
91
|
+
|
92
|
+
attrs.each do |name|
|
93
|
+
synchronized_getter = "__synchronized_#{name}"
|
94
|
+
|
95
|
+
singleton_class.instance_eval do
|
96
|
+
undef_method(name) if method_defined?(name) || private_method_defined?(name)
|
97
|
+
end
|
98
|
+
|
99
|
+
define_singleton_method(synchronized_getter) { nil }
|
100
|
+
singleton_class.class_eval do
|
101
|
+
private(synchronized_getter)
|
102
|
+
end
|
103
|
+
|
104
|
+
define_singleton_method(name) { ACCESSOR_MUTEX.synchronize { send synchronized_getter } }
|
105
|
+
|
106
|
+
ivar = "@#{name}"
|
107
|
+
|
108
|
+
singleton_class.instance_eval do
|
109
|
+
m = "#{name}="
|
110
|
+
undef_method(m) if method_defined?(m) || private_method_defined?(m)
|
111
|
+
end
|
112
|
+
define_singleton_method("#{name}=") do |val|
|
113
|
+
singleton_class.class_eval do
|
114
|
+
ACCESSOR_MUTEX.synchronize do
|
115
|
+
undef_method(synchronized_getter) if method_defined?(synchronized_getter) || private_method_defined?(synchronized_getter)
|
116
|
+
define_method(synchronized_getter) { val }
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
if singleton_class?
|
121
|
+
class_eval do
|
122
|
+
undef_method(name) if method_defined?(name) || private_method_defined?(name)
|
123
|
+
define_method(name) do
|
124
|
+
if instance_variable_defined? ivar
|
125
|
+
instance_variable_get ivar
|
126
|
+
else
|
127
|
+
singleton_class.send name
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
end
|
132
|
+
val
|
133
|
+
end
|
134
|
+
|
135
|
+
if instance_reader
|
136
|
+
undef_method(name) if method_defined?(name) || private_method_defined?(name)
|
137
|
+
define_method(name) do
|
138
|
+
if instance_variable_defined?(ivar)
|
139
|
+
instance_variable_get ivar
|
140
|
+
else
|
141
|
+
self.class.public_send name
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
if instance_writer
|
147
|
+
m = "#{name}="
|
148
|
+
undef_method(m) if method_defined?(m) || private_method_defined?(m)
|
149
|
+
attr_writer name
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
24
156
|
attr_accessor :jid
|
25
157
|
|
26
158
|
def self.included(base)
|
27
|
-
raise ArgumentError, "
|
159
|
+
raise ArgumentError, "Sidekiq::Worker cannot be included in an ActiveJob: #{base.name}" if base.ancestors.any? { |c| c.name == "ActiveJob::Base" }
|
28
160
|
|
161
|
+
base.include(Options)
|
29
162
|
base.extend(ClassMethods)
|
30
|
-
base.sidekiq_class_attribute :sidekiq_options_hash
|
31
|
-
base.sidekiq_class_attribute :sidekiq_retry_in_block
|
32
|
-
base.sidekiq_class_attribute :sidekiq_retries_exhausted_block
|
33
163
|
end
|
34
164
|
|
35
165
|
def logger
|
@@ -41,38 +171,100 @@ module Sidekiq
|
|
41
171
|
# SomeWorker.set(queue: 'foo').perform_async(....)
|
42
172
|
#
|
43
173
|
class Setter
|
174
|
+
include Sidekiq::JobUtil
|
175
|
+
|
44
176
|
def initialize(klass, opts)
|
45
177
|
@klass = klass
|
46
|
-
|
178
|
+
# NB: the internal hash always has stringified keys
|
179
|
+
@opts = opts.transform_keys(&:to_s)
|
180
|
+
|
181
|
+
# ActiveJob compatibility
|
182
|
+
interval = @opts.delete("wait_until") || @opts.delete("wait")
|
183
|
+
at(interval) if interval
|
47
184
|
end
|
48
185
|
|
49
186
|
def set(options)
|
50
|
-
|
187
|
+
hash = options.transform_keys(&:to_s)
|
188
|
+
interval = hash.delete("wait_until") || @opts.delete("wait")
|
189
|
+
@opts.merge!(hash)
|
190
|
+
at(interval) if interval
|
51
191
|
self
|
52
192
|
end
|
53
193
|
|
54
194
|
def perform_async(*args)
|
55
|
-
@
|
195
|
+
if @opts["sync"] == true
|
196
|
+
perform_inline(*args)
|
197
|
+
else
|
198
|
+
@klass.client_push(@opts.merge("args" => args, "class" => @klass))
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
# Explicit inline execution of a job. Returns nil if the job did not
|
203
|
+
# execute, true otherwise.
|
204
|
+
def perform_inline(*args)
|
205
|
+
raw = @opts.merge("args" => args, "class" => @klass)
|
206
|
+
|
207
|
+
# validate and normalize payload
|
208
|
+
item = normalize_item(raw)
|
209
|
+
queue = item["queue"]
|
210
|
+
|
211
|
+
# run client-side middleware
|
212
|
+
result = Sidekiq.client_middleware.invoke(item["class"], item, queue, Sidekiq.redis_pool) do
|
213
|
+
item
|
214
|
+
end
|
215
|
+
return nil unless result
|
216
|
+
|
217
|
+
# round-trip the payload via JSON
|
218
|
+
msg = Sidekiq.load_json(Sidekiq.dump_json(item))
|
219
|
+
|
220
|
+
# prepare the job instance
|
221
|
+
klass = msg["class"].constantize
|
222
|
+
job = klass.new
|
223
|
+
job.jid = msg["jid"]
|
224
|
+
job.bid = msg["bid"] if job.respond_to?(:bid)
|
225
|
+
|
226
|
+
# run the job through server-side middleware
|
227
|
+
result = Sidekiq.server_middleware.invoke(job, msg, msg["queue"]) do
|
228
|
+
# perform it
|
229
|
+
job.perform(*msg["args"])
|
230
|
+
true
|
231
|
+
end
|
232
|
+
return nil unless result
|
233
|
+
# jobs do not return a result. they should store any
|
234
|
+
# modified state.
|
235
|
+
true
|
236
|
+
end
|
237
|
+
alias_method :perform_sync, :perform_inline
|
238
|
+
|
239
|
+
def perform_bulk(args, batch_size: 1_000)
|
240
|
+
client = @klass.build_client
|
241
|
+
result = args.each_slice(batch_size).flat_map do |slice|
|
242
|
+
client.push_bulk(@opts.merge("class" => @klass, "args" => slice))
|
243
|
+
end
|
244
|
+
|
245
|
+
result.is_a?(Enumerator::Lazy) ? result.force : result
|
56
246
|
end
|
57
247
|
|
58
248
|
# +interval+ must be a timestamp, numeric or something that acts
|
59
249
|
# numeric (like an activesupport time interval).
|
60
250
|
def perform_in(interval, *args)
|
251
|
+
at(interval).perform_async(*args)
|
252
|
+
end
|
253
|
+
alias_method :perform_at, :perform_in
|
254
|
+
|
255
|
+
private
|
256
|
+
|
257
|
+
def at(interval)
|
61
258
|
int = interval.to_f
|
62
259
|
now = Time.now.to_f
|
63
260
|
ts = (int < 1_000_000_000 ? now + int : int)
|
64
|
-
|
65
|
-
payload = @opts.merge('class' => @klass, 'args' => args, 'at' => ts)
|
66
261
|
# Optimization to enqueue something now that is scheduled to go out now or in the past
|
67
|
-
|
68
|
-
|
262
|
+
@opts["at"] = ts if ts > now
|
263
|
+
self
|
69
264
|
end
|
70
|
-
alias_method :perform_at, :perform_in
|
71
265
|
end
|
72
266
|
|
73
267
|
module ClassMethods
|
74
|
-
ACCESSOR_MUTEX = Mutex.new
|
75
|
-
|
76
268
|
def delay(*args)
|
77
269
|
raise ArgumentError, "Do not call .delay on a Sidekiq::Worker class, call .perform_async"
|
78
270
|
end
|
@@ -85,12 +277,46 @@ module Sidekiq
|
|
85
277
|
raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
|
86
278
|
end
|
87
279
|
|
280
|
+
def queue_as(q)
|
281
|
+
sidekiq_options("queue" => q.to_s)
|
282
|
+
end
|
283
|
+
|
88
284
|
def set(options)
|
89
285
|
Setter.new(self, options)
|
90
286
|
end
|
91
287
|
|
92
288
|
def perform_async(*args)
|
93
|
-
|
289
|
+
Setter.new(self, {}).perform_async(*args)
|
290
|
+
end
|
291
|
+
|
292
|
+
# Inline execution of job's perform method after passing through Sidekiq.client_middleware and Sidekiq.server_middleware
|
293
|
+
def perform_inline(*args)
|
294
|
+
Setter.new(self, {}).perform_inline(*args)
|
295
|
+
end
|
296
|
+
alias_method :perform_sync, :perform_inline
|
297
|
+
|
298
|
+
##
|
299
|
+
# Push a large number of jobs to Redis, while limiting the batch of
|
300
|
+
# each job payload to 1,000. This method helps cut down on the number
|
301
|
+
# of round trips to Redis, which can increase the performance of enqueueing
|
302
|
+
# large numbers of jobs.
|
303
|
+
#
|
304
|
+
# +items+ must be an Array of Arrays.
|
305
|
+
#
|
306
|
+
# For finer-grained control, use `Sidekiq::Client.push_bulk` directly.
|
307
|
+
#
|
308
|
+
# Example (3 Redis round trips):
|
309
|
+
#
|
310
|
+
# SomeWorker.perform_async(1)
|
311
|
+
# SomeWorker.perform_async(2)
|
312
|
+
# SomeWorker.perform_async(3)
|
313
|
+
#
|
314
|
+
# Would instead become (1 Redis round trip):
|
315
|
+
#
|
316
|
+
# SomeWorker.perform_bulk([[1], [2], [3]])
|
317
|
+
#
|
318
|
+
def perform_bulk(*args, **kwargs)
|
319
|
+
Setter.new(self, {}).perform_bulk(*args, **kwargs)
|
94
320
|
end
|
95
321
|
|
96
322
|
# +interval+ must be a timestamp, numeric or something that acts
|
@@ -100,10 +326,10 @@ module Sidekiq
|
|
100
326
|
now = Time.now.to_f
|
101
327
|
ts = (int < 1_000_000_000 ? now + int : int)
|
102
328
|
|
103
|
-
item = {
|
329
|
+
item = {"class" => self, "args" => args}
|
104
330
|
|
105
331
|
# Optimization to enqueue something now that is scheduled to go out now or in the past
|
106
|
-
item
|
332
|
+
item["at"] = ts if ts > now
|
107
333
|
|
108
334
|
client_push(item)
|
109
335
|
end
|
@@ -122,99 +348,20 @@ module Sidekiq
|
|
122
348
|
#
|
123
349
|
# In practice, any option is allowed. This is the main mechanism to configure the
|
124
350
|
# options for a specific job.
|
125
|
-
def sidekiq_options(opts={})
|
126
|
-
|
127
|
-
self.sidekiq_options_hash = get_sidekiq_options.merge(Hash[opts.map{|k, v| [k.to_s, v]}])
|
128
|
-
end
|
129
|
-
|
130
|
-
def sidekiq_retry_in(&block)
|
131
|
-
self.sidekiq_retry_in_block = block
|
132
|
-
end
|
133
|
-
|
134
|
-
def sidekiq_retries_exhausted(&block)
|
135
|
-
self.sidekiq_retries_exhausted_block = block
|
136
|
-
end
|
137
|
-
|
138
|
-
def get_sidekiq_options # :nodoc:
|
139
|
-
self.sidekiq_options_hash ||= Sidekiq.default_worker_options
|
351
|
+
def sidekiq_options(opts = {})
|
352
|
+
super
|
140
353
|
end
|
141
354
|
|
142
355
|
def client_push(item) # :nodoc:
|
143
|
-
|
144
|
-
|
145
|
-
item.keys.each do |key|
|
146
|
-
item[key.to_s] = item.delete(key)
|
147
|
-
end
|
148
|
-
|
149
|
-
Sidekiq::Client.new(pool).push(item)
|
356
|
+
raise ArgumentError, "Job payloads should contain no Symbols: #{item}" if item.any? { |k, v| k.is_a?(::Symbol) }
|
357
|
+
build_client.push(item)
|
150
358
|
end
|
151
359
|
|
152
|
-
def
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
attrs.each do |name|
|
157
|
-
synchronized_getter = "__synchronized_#{name}"
|
158
|
-
|
159
|
-
singleton_class.instance_eval do
|
160
|
-
undef_method(name) if method_defined?(name) || private_method_defined?(name)
|
161
|
-
end
|
162
|
-
|
163
|
-
define_singleton_method(synchronized_getter) { nil }
|
164
|
-
singleton_class.class_eval do
|
165
|
-
private(synchronized_getter)
|
166
|
-
end
|
167
|
-
|
168
|
-
define_singleton_method(name) { ACCESSOR_MUTEX.synchronize { send synchronized_getter } }
|
169
|
-
|
170
|
-
ivar = "@#{name}"
|
171
|
-
|
172
|
-
singleton_class.instance_eval do
|
173
|
-
m = "#{name}="
|
174
|
-
undef_method(m) if method_defined?(m) || private_method_defined?(m)
|
175
|
-
end
|
176
|
-
define_singleton_method("#{name}=") do |val|
|
177
|
-
singleton_class.class_eval do
|
178
|
-
ACCESSOR_MUTEX.synchronize do
|
179
|
-
undef_method(synchronized_getter) if method_defined?(synchronized_getter) || private_method_defined?(synchronized_getter)
|
180
|
-
define_method(synchronized_getter) { val }
|
181
|
-
end
|
182
|
-
end
|
183
|
-
|
184
|
-
if singleton_class?
|
185
|
-
class_eval do
|
186
|
-
undef_method(name) if method_defined?(name) || private_method_defined?(name)
|
187
|
-
define_method(name) do
|
188
|
-
if instance_variable_defined? ivar
|
189
|
-
instance_variable_get ivar
|
190
|
-
else
|
191
|
-
singleton_class.send name
|
192
|
-
end
|
193
|
-
end
|
194
|
-
end
|
195
|
-
end
|
196
|
-
val
|
197
|
-
end
|
198
|
-
|
199
|
-
if instance_reader
|
200
|
-
undef_method(name) if method_defined?(name) || private_method_defined?(name)
|
201
|
-
define_method(name) do
|
202
|
-
if instance_variable_defined?(ivar)
|
203
|
-
instance_variable_get ivar
|
204
|
-
else
|
205
|
-
self.class.public_send name
|
206
|
-
end
|
207
|
-
end
|
208
|
-
end
|
209
|
-
|
210
|
-
if instance_writer
|
211
|
-
m = "#{name}="
|
212
|
-
undef_method(m) if method_defined?(m) || private_method_defined?(m)
|
213
|
-
attr_writer name
|
214
|
-
end
|
215
|
-
end
|
360
|
+
def build_client # :nodoc:
|
361
|
+
pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options["pool"] || Sidekiq.redis_pool
|
362
|
+
client_class = get_sidekiq_options["client_class"] || Sidekiq::Client
|
363
|
+
client_class.new(pool)
|
216
364
|
end
|
217
|
-
|
218
365
|
end
|
219
366
|
end
|
220
367
|
end
|