sidekiq 3.5.4 → 7.2.0
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +992 -6
- data/LICENSE.txt +9 -0
- data/README.md +52 -43
- data/bin/sidekiq +22 -4
- data/bin/sidekiqload +209 -115
- data/bin/sidekiqmon +11 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/job_spec.rb.erb +6 -0
- data/lib/generators/sidekiq/templates/job_test.rb.erb +8 -0
- data/lib/sidekiq/api.rb +633 -295
- data/lib/sidekiq/capsule.rb +127 -0
- data/lib/sidekiq/cli.rb +270 -248
- data/lib/sidekiq/client.rb +139 -108
- data/lib/sidekiq/component.rb +68 -0
- data/lib/sidekiq/config.rb +287 -0
- data/lib/sidekiq/deploy.rb +62 -0
- data/lib/sidekiq/embedded.rb +61 -0
- data/lib/sidekiq/fetch.rb +53 -121
- data/lib/sidekiq/job.rb +374 -0
- data/lib/sidekiq/job_logger.rb +51 -0
- data/lib/sidekiq/job_retry.rb +301 -0
- data/lib/sidekiq/job_util.rb +107 -0
- data/lib/sidekiq/launcher.rb +241 -69
- data/lib/sidekiq/logger.rb +131 -0
- data/lib/sidekiq/manager.rb +88 -190
- data/lib/sidekiq/metrics/query.rb +155 -0
- data/lib/sidekiq/metrics/shared.rb +95 -0
- data/lib/sidekiq/metrics/tracking.rb +136 -0
- data/lib/sidekiq/middleware/chain.rb +114 -56
- data/lib/sidekiq/middleware/current_attributes.rb +95 -0
- data/lib/sidekiq/middleware/i18n.rb +8 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +146 -0
- data/lib/sidekiq/paginator.rb +29 -16
- data/lib/sidekiq/processor.rb +238 -118
- data/lib/sidekiq/rails.rb +57 -27
- data/lib/sidekiq/redis_client_adapter.rb +111 -0
- data/lib/sidekiq/redis_connection.rb +49 -50
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +173 -52
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +7 -5
- data/lib/sidekiq/testing.rb +197 -65
- data/lib/sidekiq/transaction_aware_client.rb +44 -0
- data/lib/sidekiq/version.rb +4 -1
- data/lib/sidekiq/web/action.rb +93 -0
- data/lib/sidekiq/web/application.rb +463 -0
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +364 -0
- data/lib/sidekiq/web/router.rb +104 -0
- data/lib/sidekiq/web.rb +113 -216
- data/lib/sidekiq/worker_compatibility_alias.rb +13 -0
- data/lib/sidekiq.rb +99 -142
- data/sidekiq.gemspec +26 -23
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +163 -74
- data/web/assets/javascripts/base-charts.js +106 -0
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +182 -0
- data/web/assets/javascripts/dashboard.js +37 -280
- data/web/assets/javascripts/metrics.js +298 -0
- data/web/assets/stylesheets/application-dark.css +147 -0
- data/web/assets/stylesheets/application-rtl.css +153 -0
- data/web/assets/stylesheets/application.css +181 -198
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +4 -8
- data/web/locales/ar.yml +87 -0
- data/web/locales/cs.yml +62 -52
- data/web/locales/da.yml +60 -53
- data/web/locales/de.yml +65 -53
- data/web/locales/el.yml +43 -24
- data/web/locales/en.yml +86 -62
- data/web/locales/es.yml +70 -53
- data/web/locales/fa.yml +80 -0
- data/web/locales/fr.yml +86 -56
- data/web/locales/gd.yml +99 -0
- data/web/locales/he.yml +80 -0
- data/web/locales/hi.yml +59 -59
- data/web/locales/it.yml +53 -53
- data/web/locales/ja.yml +78 -56
- data/web/locales/ko.yml +52 -52
- data/web/locales/lt.yml +83 -0
- data/web/locales/nb.yml +61 -61
- data/web/locales/nl.yml +52 -52
- data/web/locales/pl.yml +45 -45
- data/web/locales/pt-br.yml +83 -55
- data/web/locales/pt.yml +51 -51
- data/web/locales/ru.yml +68 -60
- data/web/locales/sv.yml +53 -53
- data/web/locales/ta.yml +60 -60
- data/web/locales/uk.yml +62 -61
- data/web/locales/ur.yml +80 -0
- data/web/locales/vi.yml +83 -0
- data/web/locales/zh-cn.yml +43 -16
- data/web/locales/zh-tw.yml +42 -8
- data/web/views/_footer.erb +10 -9
- data/web/views/_job_info.erb +26 -5
- data/web/views/_metrics_period_select.erb +12 -0
- data/web/views/_nav.erb +6 -20
- data/web/views/_paging.erb +3 -1
- data/web/views/_poll_link.erb +3 -6
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +87 -28
- data/web/views/dashboard.erb +51 -21
- data/web/views/dead.erb +4 -4
- data/web/views/filtering.erb +7 -0
- data/web/views/layout.erb +15 -5
- data/web/views/metrics.erb +91 -0
- data/web/views/metrics_for_job.erb +59 -0
- data/web/views/morgue.erb +25 -22
- data/web/views/queue.erb +35 -25
- data/web/views/queues.erb +23 -7
- data/web/views/retries.erb +28 -23
- data/web/views/retry.erb +5 -5
- data/web/views/scheduled.erb +19 -17
- data/web/views/scheduled_job_info.erb +1 -1
- metadata +86 -268
- data/.gitignore +0 -12
- data/.travis.yml +0 -16
- data/3.0-Upgrade.md +0 -70
- data/COMM-LICENSE +0 -95
- data/Contributing.md +0 -32
- data/Ent-Changes.md +0 -39
- data/Gemfile +0 -27
- data/LICENSE +0 -9
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-Changes.md +0 -454
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -93
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +0 -6
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +0 -8
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/actor.rb +0 -39
- data/lib/sidekiq/core_ext.rb +0 -105
- data/lib/sidekiq/exception_handler.rb +0 -30
- data/lib/sidekiq/extensions/action_mailer.rb +0 -56
- data/lib/sidekiq/extensions/active_record.rb +0 -39
- data/lib/sidekiq/extensions/class_methods.rb +0 -39
- data/lib/sidekiq/extensions/generic_proxy.rb +0 -24
- data/lib/sidekiq/logging.rb +0 -104
- data/lib/sidekiq/middleware/server/active_record.rb +0 -13
- data/lib/sidekiq/middleware/server/logging.rb +0 -40
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -206
- data/lib/sidekiq/util.rb +0 -68
- data/lib/sidekiq/web_helpers.rb +0 -249
- data/lib/sidekiq/worker.rb +0 -103
- data/test/config.yml +0 -9
- data/test/env_based_config.yml +0 -11
- data/test/fake_env.rb +0 -0
- data/test/fixtures/en.yml +0 -2
- data/test/helper.rb +0 -49
- data/test/test_api.rb +0 -493
- data/test/test_cli.rb +0 -335
- data/test/test_client.rb +0 -194
- data/test/test_exception_handler.rb +0 -55
- data/test/test_extensions.rb +0 -126
- data/test/test_fetch.rb +0 -104
- data/test/test_logging.rb +0 -34
- data/test/test_manager.rb +0 -168
- data/test/test_middleware.rb +0 -159
- data/test/test_processor.rb +0 -237
- data/test/test_rails.rb +0 -21
- data/test/test_redis_connection.rb +0 -126
- data/test/test_retry.rb +0 -325
- data/test/test_scheduled.rb +0 -114
- data/test/test_scheduling.rb +0 -49
- data/test/test_sidekiq.rb +0 -99
- data/test/test_testing.rb +0 -142
- data/test/test_testing_fake.rb +0 -268
- data/test/test_testing_inline.rb +0 -93
- data/test/test_util.rb +0 -16
- data/test/test_web.rb +0 -608
- data/test/test_web_helpers.rb +0 -53
- data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
- data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
- data/web/assets/images/status/active.png +0 -0
- data/web/assets/images/status/idle.png +0 -0
- data/web/assets/javascripts/locales/README.md +0 -27
- data/web/assets/javascripts/locales/jquery.timeago.ar.js +0 -96
- data/web/assets/javascripts/locales/jquery.timeago.bg.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.bs.js +0 -49
- data/web/assets/javascripts/locales/jquery.timeago.ca.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.cs.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.cy.js +0 -20
- data/web/assets/javascripts/locales/jquery.timeago.da.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.de.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.el.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.en-short.js +0 -20
- data/web/assets/javascripts/locales/jquery.timeago.en.js +0 -20
- data/web/assets/javascripts/locales/jquery.timeago.es.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.et.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.fa.js +0 -22
- data/web/assets/javascripts/locales/jquery.timeago.fi.js +0 -28
- data/web/assets/javascripts/locales/jquery.timeago.fr-short.js +0 -16
- data/web/assets/javascripts/locales/jquery.timeago.fr.js +0 -17
- data/web/assets/javascripts/locales/jquery.timeago.he.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.hr.js +0 -49
- data/web/assets/javascripts/locales/jquery.timeago.hu.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.hy.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.id.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.it.js +0 -16
- data/web/assets/javascripts/locales/jquery.timeago.ja.js +0 -19
- data/web/assets/javascripts/locales/jquery.timeago.ko.js +0 -17
- data/web/assets/javascripts/locales/jquery.timeago.lt.js +0 -20
- data/web/assets/javascripts/locales/jquery.timeago.mk.js +0 -20
- data/web/assets/javascripts/locales/jquery.timeago.nl.js +0 -20
- data/web/assets/javascripts/locales/jquery.timeago.no.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.pl.js +0 -31
- data/web/assets/javascripts/locales/jquery.timeago.pt-br.js +0 -16
- data/web/assets/javascripts/locales/jquery.timeago.pt.js +0 -16
- data/web/assets/javascripts/locales/jquery.timeago.ro.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.rs.js +0 -49
- data/web/assets/javascripts/locales/jquery.timeago.ru.js +0 -34
- data/web/assets/javascripts/locales/jquery.timeago.sk.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.sl.js +0 -44
- data/web/assets/javascripts/locales/jquery.timeago.sv.js +0 -18
- data/web/assets/javascripts/locales/jquery.timeago.th.js +0 -20
- data/web/assets/javascripts/locales/jquery.timeago.tr.js +0 -16
- data/web/assets/javascripts/locales/jquery.timeago.uk.js +0 -34
- data/web/assets/javascripts/locales/jquery.timeago.uz.js +0 -19
- data/web/assets/javascripts/locales/jquery.timeago.zh-cn.js +0 -20
- data/web/assets/javascripts/locales/jquery.timeago.zh-tw.js +0 -20
- data/web/views/_poll_js.erb +0 -5
- /data/web/assets/images/{status-sd8051fd480.png → status.png} +0 -0
data/lib/sidekiq/paginator.rb
CHANGED
@@ -1,8 +1,9 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Sidekiq
|
2
4
|
module Paginator
|
3
|
-
|
4
|
-
|
5
|
-
current_page = pageidx.to_i < 1 ? 1 : pageidx.to_i
|
5
|
+
def page(key, pageidx = 1, page_size = 25, opts = nil)
|
6
|
+
current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
|
6
7
|
pageidx = current_page - 1
|
7
8
|
total_size = 0
|
8
9
|
items = []
|
@@ -11,26 +12,31 @@ module Sidekiq
|
|
11
12
|
|
12
13
|
Sidekiq.redis do |conn|
|
13
14
|
type = conn.type(key)
|
15
|
+
rev = opts && opts[:reverse]
|
14
16
|
|
15
17
|
case type
|
16
|
-
when
|
17
|
-
|
18
|
-
|
19
|
-
conn.zcard(key)
|
18
|
+
when "zset"
|
19
|
+
total_size, items = conn.multi { |transaction|
|
20
|
+
transaction.zcard(key)
|
20
21
|
if rev
|
21
|
-
|
22
|
+
transaction.zrange(key, starting, ending, "REV", "withscores")
|
22
23
|
else
|
23
|
-
|
24
|
+
transaction.zrange(key, starting, ending, "withscores")
|
24
25
|
end
|
25
|
-
|
26
|
+
}
|
26
27
|
[current_page, total_size, items]
|
27
|
-
when
|
28
|
-
total_size, items = conn.multi
|
29
|
-
|
30
|
-
|
31
|
-
|
28
|
+
when "list"
|
29
|
+
total_size, items = conn.multi { |transaction|
|
30
|
+
transaction.llen(key)
|
31
|
+
if rev
|
32
|
+
transaction.lrange(key, -ending - 1, -starting - 1)
|
33
|
+
else
|
34
|
+
transaction.lrange(key, starting, ending)
|
35
|
+
end
|
36
|
+
}
|
37
|
+
items.reverse! if rev
|
32
38
|
[current_page, total_size, items]
|
33
|
-
when
|
39
|
+
when "none"
|
34
40
|
[1, 0, []]
|
35
41
|
else
|
36
42
|
raise "can't page a #{type}"
|
@@ -38,5 +44,12 @@ module Sidekiq
|
|
38
44
|
end
|
39
45
|
end
|
40
46
|
|
47
|
+
def page_items(items, pageidx = 1, page_size = 25)
|
48
|
+
current_page = (pageidx.to_i < 1) ? 1 : pageidx.to_i
|
49
|
+
pageidx = current_page - 1
|
50
|
+
starting = pageidx * page_size
|
51
|
+
items = items.to_a
|
52
|
+
[current_page, items.size, items[starting, page_size]]
|
53
|
+
end
|
41
54
|
end
|
42
55
|
end
|
data/lib/sidekiq/processor.rb
CHANGED
@@ -1,161 +1,281 @@
|
|
1
|
-
|
2
|
-
require 'sidekiq/actor'
|
1
|
+
# frozen_string_literal: true
|
3
2
|
|
4
|
-
require
|
5
|
-
require
|
3
|
+
require "sidekiq/fetch"
|
4
|
+
require "sidekiq/job_logger"
|
5
|
+
require "sidekiq/job_retry"
|
6
6
|
|
7
7
|
module Sidekiq
|
8
8
|
##
|
9
|
-
# The Processor
|
10
|
-
#
|
11
|
-
#
|
9
|
+
# The Processor is a standalone thread which:
|
10
|
+
#
|
11
|
+
# 1. fetches a job from Redis
|
12
|
+
# 2. executes the job
|
13
|
+
# a. instantiate the job class
|
14
|
+
# b. run the middleware chain
|
15
|
+
# c. call #perform
|
16
|
+
#
|
17
|
+
# A Processor can exit due to shutdown or due to
|
18
|
+
# an error during job execution.
|
19
|
+
#
|
20
|
+
# If an error occurs in the job execution, the
|
21
|
+
# Processor calls the Manager to create a new one
|
22
|
+
# to replace itself and exits.
|
23
|
+
#
|
12
24
|
class Processor
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
25
|
+
include Sidekiq::Component
|
26
|
+
|
27
|
+
attr_reader :thread
|
28
|
+
attr_reader :job
|
29
|
+
attr_reader :capsule
|
30
|
+
|
31
|
+
def initialize(capsule, &block)
|
32
|
+
@config = @capsule = capsule
|
33
|
+
@callback = block
|
34
|
+
@down = false
|
35
|
+
@done = false
|
36
|
+
@job = nil
|
37
|
+
@thread = nil
|
38
|
+
@reloader = Sidekiq.default_configuration[:reloader]
|
39
|
+
@job_logger = (capsule.config[:job_logger] || Sidekiq::JobLogger).new(logger)
|
40
|
+
@retrier = Sidekiq::JobRetry.new(capsule)
|
29
41
|
end
|
30
42
|
|
31
|
-
|
43
|
+
def terminate(wait = false)
|
44
|
+
@done = true
|
45
|
+
return unless @thread
|
46
|
+
@thread.value if wait
|
47
|
+
end
|
32
48
|
|
33
|
-
def
|
34
|
-
@
|
49
|
+
def kill(wait = false)
|
50
|
+
@done = true
|
51
|
+
return unless @thread
|
52
|
+
# unlike the other actors, terminate does not wait
|
53
|
+
# for the thread to finish because we don't know how
|
54
|
+
# long the job will take to finish. Instead we
|
55
|
+
# provide a `kill` method to call after the shutdown
|
56
|
+
# timeout passes.
|
57
|
+
@thread.raise ::Sidekiq::Shutdown
|
58
|
+
@thread.value if wait
|
35
59
|
end
|
36
60
|
|
37
|
-
def
|
38
|
-
|
39
|
-
|
61
|
+
def start
|
62
|
+
@thread ||= safe_thread("#{config.name}/processor", &method(:run))
|
63
|
+
end
|
40
64
|
|
41
|
-
|
65
|
+
private unless $TESTING
|
42
66
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
worker = klass.new
|
48
|
-
worker.jid = msg['jid'.freeze]
|
49
|
-
|
50
|
-
stats(worker, msg, queue) do
|
51
|
-
Sidekiq.server_middleware.invoke(worker, msg, queue) do
|
52
|
-
# Only ack if we either attempted to start this job or
|
53
|
-
# successfully completed it. This prevents us from
|
54
|
-
# losing jobs if a middleware raises an exception before yielding
|
55
|
-
ack = true
|
56
|
-
execute_job(worker, cloned(msg['args'.freeze]))
|
57
|
-
end
|
58
|
-
end
|
59
|
-
ack = true
|
60
|
-
rescue Sidekiq::Shutdown
|
61
|
-
# Had to force kill this job because it didn't finish
|
62
|
-
# within the timeout. Don't acknowledge the work since
|
63
|
-
# we didn't properly finish it.
|
64
|
-
ack = false
|
65
|
-
rescue Exception => ex
|
66
|
-
handle_exception(ex, msg || { :message => msgstr })
|
67
|
-
raise
|
68
|
-
ensure
|
69
|
-
work.acknowledge if ack
|
70
|
-
end
|
67
|
+
def run
|
68
|
+
# By setting this thread-local, Sidekiq.redis will access +Sidekiq::Capsule#redis_pool+
|
69
|
+
# instead of the global pool in +Sidekiq::Config#redis_pool+.
|
70
|
+
Thread.current[:sidekiq_capsule] = @capsule
|
71
71
|
|
72
|
-
@
|
72
|
+
process_one until @done
|
73
|
+
@callback.call(self)
|
74
|
+
rescue Sidekiq::Shutdown
|
75
|
+
@callback.call(self)
|
76
|
+
rescue Exception => ex
|
77
|
+
@callback.call(self, ex)
|
73
78
|
end
|
74
79
|
|
75
|
-
def
|
76
|
-
|
80
|
+
def process_one(&block)
|
81
|
+
@job = fetch
|
82
|
+
process(@job) if @job
|
83
|
+
@job = nil
|
77
84
|
end
|
78
85
|
|
79
|
-
def
|
80
|
-
|
86
|
+
def get_one
|
87
|
+
uow = capsule.fetcher.retrieve_work
|
88
|
+
if @down
|
89
|
+
logger.info { "Redis is online, #{::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @down} sec downtime" }
|
90
|
+
@down = nil
|
91
|
+
end
|
92
|
+
uow
|
93
|
+
rescue Sidekiq::Shutdown
|
94
|
+
rescue => ex
|
95
|
+
handle_fetch_exception(ex)
|
81
96
|
end
|
82
97
|
|
83
|
-
|
98
|
+
def fetch
|
99
|
+
j = get_one
|
100
|
+
if j && @done
|
101
|
+
j.requeue
|
102
|
+
nil
|
103
|
+
else
|
104
|
+
j
|
105
|
+
end
|
106
|
+
end
|
84
107
|
|
85
|
-
def
|
86
|
-
@
|
108
|
+
def handle_fetch_exception(ex)
|
109
|
+
unless @down
|
110
|
+
@down = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
111
|
+
logger.error("Error fetching job: #{ex}")
|
112
|
+
handle_exception(ex)
|
113
|
+
end
|
114
|
+
sleep(1)
|
115
|
+
nil
|
87
116
|
end
|
88
117
|
|
89
|
-
def
|
90
|
-
#
|
91
|
-
#
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
118
|
+
def dispatch(job_hash, queue, jobstr)
|
119
|
+
# since middleware can mutate the job hash
|
120
|
+
# we need to clone it to report the original
|
121
|
+
# job structure to the Web UI
|
122
|
+
# or to push back to redis when retrying.
|
123
|
+
# To avoid costly and, most of the time, useless cloning here,
|
124
|
+
# we pass original String of JSON to respected methods
|
125
|
+
# to re-parse it there if we need access to the original, untouched job
|
126
|
+
|
127
|
+
@job_logger.prepare(job_hash) do
|
128
|
+
@retrier.global(jobstr, queue) do
|
129
|
+
@job_logger.call(job_hash, queue) do
|
130
|
+
stats(jobstr, queue) do
|
131
|
+
# Rails 5 requires a Reloader to wrap code execution. In order to
|
132
|
+
# constantize the worker and instantiate an instance, we have to call
|
133
|
+
# the Reloader. It handles code loading, db connection management, etc.
|
134
|
+
# Effectively this block denotes a "unit of work" to Rails.
|
135
|
+
@reloader.call do
|
136
|
+
klass = Object.const_get(job_hash["class"])
|
137
|
+
inst = klass.new
|
138
|
+
inst.jid = job_hash["jid"]
|
139
|
+
@retrier.local(inst, jobstr, queue) do
|
140
|
+
yield inst
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
98
144
|
end
|
99
145
|
end
|
100
146
|
end
|
147
|
+
end
|
148
|
+
|
149
|
+
IGNORE_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :never}
|
150
|
+
private_constant :IGNORE_SHUTDOWN_INTERRUPTS
|
151
|
+
ALLOW_SHUTDOWN_INTERRUPTS = {Sidekiq::Shutdown => :immediate}
|
152
|
+
private_constant :ALLOW_SHUTDOWN_INTERRUPTS
|
101
153
|
|
102
|
-
|
154
|
+
def process(uow)
|
155
|
+
jobstr = uow.job
|
156
|
+
queue = uow.queue_name
|
157
|
+
|
158
|
+
# Treat malformed JSON as a special case: job goes straight to the morgue.
|
159
|
+
job_hash = nil
|
103
160
|
begin
|
104
|
-
|
105
|
-
rescue
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
end
|
161
|
+
job_hash = Sidekiq.load_json(jobstr)
|
162
|
+
rescue => ex
|
163
|
+
handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
|
164
|
+
now = Time.now.to_f
|
165
|
+
redis do |conn|
|
166
|
+
conn.multi do |xa|
|
167
|
+
xa.zadd("dead", now.to_s, jobstr)
|
168
|
+
xa.zremrangebyscore("dead", "-inf", now - @capsule.config[:dead_timeout_in_seconds])
|
169
|
+
xa.zremrangebyrank("dead", 0, - @capsule.config[:dead_max_jobs])
|
114
170
|
end
|
115
171
|
end
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
conn.expire(processed, STATS_TIMEOUT)
|
172
|
+
return uow.acknowledge
|
173
|
+
end
|
174
|
+
|
175
|
+
ack = false
|
176
|
+
Thread.handle_interrupt(IGNORE_SHUTDOWN_INTERRUPTS) do
|
177
|
+
Thread.handle_interrupt(ALLOW_SHUTDOWN_INTERRUPTS) do
|
178
|
+
dispatch(job_hash, queue, jobstr) do |inst|
|
179
|
+
config.server_middleware.invoke(inst, job_hash, queue) do
|
180
|
+
execute_job(inst, job_hash["args"])
|
126
181
|
end
|
127
182
|
end
|
183
|
+
ack = true
|
184
|
+
rescue Sidekiq::Shutdown
|
185
|
+
# Had to force kill this job because it didn't finish
|
186
|
+
# within the timeout. Don't acknowledge the work since
|
187
|
+
# we didn't properly finish it.
|
188
|
+
rescue Sidekiq::JobRetry::Handled => h
|
189
|
+
# this is the common case: job raised error and Sidekiq::JobRetry::Handled
|
190
|
+
# signals that we created a retry successfully. We can acknowlege the job.
|
191
|
+
ack = true
|
192
|
+
e = h.cause || h
|
193
|
+
handle_exception(e, {context: "Job raised exception", job: job_hash})
|
194
|
+
raise e
|
195
|
+
rescue Exception => ex
|
196
|
+
# Unexpected error! This is very bad and indicates an exception that got past
|
197
|
+
# the retry subsystem (e.g. network partition). We won't acknowledge the job
|
198
|
+
# so it can be rescued when using Sidekiq Pro.
|
199
|
+
handle_exception(ex, {context: "Internal exception!", job: job_hash, jobstr: jobstr})
|
200
|
+
raise ex
|
201
|
+
end
|
202
|
+
ensure
|
203
|
+
if ack
|
204
|
+
uow.acknowledge
|
128
205
|
end
|
129
206
|
end
|
130
207
|
end
|
131
208
|
|
132
|
-
|
133
|
-
|
134
|
-
# been mutated by the worker.
|
135
|
-
def cloned(ary)
|
136
|
-
Marshal.load(Marshal.dump(ary))
|
209
|
+
def execute_job(inst, cloned_args)
|
210
|
+
inst.perform(*cloned_args)
|
137
211
|
end
|
138
212
|
|
139
|
-
#
|
140
|
-
#
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
213
|
+
# Ruby doesn't provide atomic counters out of the box so we'll
|
214
|
+
# implement something simple ourselves.
|
215
|
+
# https://bugs.ruby-lang.org/issues/14706
|
216
|
+
class Counter
|
217
|
+
def initialize
|
218
|
+
@value = 0
|
219
|
+
@lock = Mutex.new
|
220
|
+
end
|
221
|
+
|
222
|
+
def incr(amount = 1)
|
223
|
+
@lock.synchronize { @value += amount }
|
224
|
+
end
|
225
|
+
|
226
|
+
def reset
|
227
|
+
@lock.synchronize {
|
228
|
+
val = @value
|
229
|
+
@value = 0
|
230
|
+
val
|
231
|
+
}
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
# jruby's Hash implementation is not threadsafe, so we wrap it in a mutex here
|
236
|
+
class SharedWorkState
|
237
|
+
def initialize
|
238
|
+
@work_state = {}
|
239
|
+
@lock = Mutex.new
|
240
|
+
end
|
241
|
+
|
242
|
+
def set(tid, hash)
|
243
|
+
@lock.synchronize { @work_state[tid] = hash }
|
244
|
+
end
|
245
|
+
|
246
|
+
def delete(tid)
|
247
|
+
@lock.synchronize { @work_state.delete(tid) }
|
248
|
+
end
|
249
|
+
|
250
|
+
def dup
|
251
|
+
@lock.synchronize { @work_state.dup }
|
252
|
+
end
|
253
|
+
|
254
|
+
def size
|
255
|
+
@lock.synchronize { @work_state.size }
|
256
|
+
end
|
257
|
+
|
258
|
+
def clear
|
259
|
+
@lock.synchronize { @work_state.clear }
|
154
260
|
end
|
155
261
|
end
|
156
262
|
|
157
|
-
|
158
|
-
|
263
|
+
PROCESSED = Counter.new
|
264
|
+
FAILURE = Counter.new
|
265
|
+
WORK_STATE = SharedWorkState.new
|
266
|
+
|
267
|
+
def stats(jobstr, queue)
|
268
|
+
WORK_STATE.set(tid, {queue: queue, payload: jobstr, run_at: Time.now.to_i})
|
269
|
+
|
270
|
+
begin
|
271
|
+
yield
|
272
|
+
rescue Exception
|
273
|
+
FAILURE.incr
|
274
|
+
raise
|
275
|
+
ensure
|
276
|
+
WORK_STATE.delete(tid)
|
277
|
+
PROCESSED.incr
|
278
|
+
end
|
159
279
|
end
|
160
280
|
end
|
161
281
|
end
|
data/lib/sidekiq/rails.rb
CHANGED
@@ -1,38 +1,68 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "sidekiq/job"
|
4
|
+
require "rails"
|
5
|
+
|
1
6
|
module Sidekiq
|
2
|
-
|
3
|
-
|
7
|
+
class Rails < ::Rails::Engine
|
8
|
+
class Reloader
|
9
|
+
def initialize(app = ::Rails.application)
|
10
|
+
@app = app
|
11
|
+
end
|
4
12
|
|
5
|
-
|
6
|
-
|
7
|
-
|
13
|
+
def call
|
14
|
+
params = (::Rails::VERSION::STRING >= "7.1") ? {source: "job.sidekiq"} : {}
|
15
|
+
@app.reloader.wrap(**params) do
|
16
|
+
yield
|
17
|
+
end
|
18
|
+
end
|
8
19
|
|
9
|
-
|
10
|
-
|
20
|
+
def inspect
|
21
|
+
"#<Sidekiq::Rails::Reloader @app=#{@app.class.name}>"
|
22
|
+
end
|
11
23
|
end
|
12
24
|
|
13
|
-
|
14
|
-
|
25
|
+
# By including the Options module, we allow AJs to directly control sidekiq features
|
26
|
+
# via the *sidekiq_options* class method and, for instance, not use AJ's retry system.
|
27
|
+
# AJ retries don't show up in the Sidekiq UI Retries tab, don't save any error data, can't be
|
28
|
+
# manually retried, don't automatically die, etc.
|
29
|
+
#
|
30
|
+
# class SomeJob < ActiveJob::Base
|
31
|
+
# queue_as :default
|
32
|
+
# sidekiq_options retry: 3, backtrace: 10
|
33
|
+
# def perform
|
34
|
+
# end
|
35
|
+
# end
|
36
|
+
initializer "sidekiq.active_job_integration" do
|
37
|
+
ActiveSupport.on_load(:active_job) do
|
38
|
+
include ::Sidekiq::Job::Options unless respond_to?(:sidekiq_options)
|
39
|
+
end
|
40
|
+
end
|
15
41
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
def self.remove_delay!
|
20
|
-
@delay_removed = true
|
21
|
-
|
22
|
-
[Extensions::ActiveRecord,
|
23
|
-
Extensions::ActionMailer,
|
24
|
-
Extensions::Klass].each do |mod|
|
25
|
-
mod.module_eval do
|
26
|
-
remove_method :delay if respond_to?(:delay)
|
27
|
-
remove_method :delay_for if respond_to?(:delay_for)
|
28
|
-
remove_method :delay_until if respond_to?(:delay_until)
|
42
|
+
initializer "sidekiq.backtrace_cleaner" do
|
43
|
+
Sidekiq.configure_server do |config|
|
44
|
+
config[:backtrace_cleaner] = ->(backtrace) { ::Rails.backtrace_cleaner.clean(backtrace) }
|
29
45
|
end
|
30
46
|
end
|
31
|
-
end
|
32
47
|
|
33
|
-
|
34
|
-
|
35
|
-
|
48
|
+
# This hook happens after all initializers are run, just before returning
|
49
|
+
# from config/environment.rb back to sidekiq/cli.rb.
|
50
|
+
#
|
51
|
+
# None of this matters on the client-side, only within the Sidekiq process itself.
|
52
|
+
config.after_initialize do
|
53
|
+
Sidekiq.configure_server do |config|
|
54
|
+
config[:reloader] = Sidekiq::Rails::Reloader.new
|
55
|
+
|
56
|
+
# This is the integration code necessary so that if a job uses `Rails.logger.info "Hello"`,
|
57
|
+
# it will appear in the Sidekiq console with all of the job context.
|
58
|
+
unless ::Rails.logger == config.logger || ::ActiveSupport::Logger.logger_outputs_to?(::Rails.logger, $stdout)
|
59
|
+
if ::Rails::VERSION::STRING < "7.1"
|
60
|
+
::Rails.logger.extend(::ActiveSupport::Logger.broadcast(config.logger))
|
61
|
+
else
|
62
|
+
::Rails.logger.broadcast_to(config.logger)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
36
66
|
end
|
37
|
-
end
|
67
|
+
end
|
38
68
|
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "set"
|
4
|
+
require "redis_client"
|
5
|
+
require "redis_client/decorator"
|
6
|
+
|
7
|
+
module Sidekiq
|
8
|
+
class RedisClientAdapter
|
9
|
+
BaseError = RedisClient::Error
|
10
|
+
CommandError = RedisClient::CommandError
|
11
|
+
|
12
|
+
# You can add/remove items or clear the whole thing if you don't want deprecation warnings.
|
13
|
+
DEPRECATED_COMMANDS = %i[rpoplpush zrangebyscore zrevrange zrevrangebyscore getset hmset setex setnx].to_set
|
14
|
+
|
15
|
+
module CompatMethods
|
16
|
+
def info
|
17
|
+
@client.call("INFO") { |i| i.lines(chomp: true).map { |l| l.split(":", 2) }.select { |l| l.size == 2 }.to_h }
|
18
|
+
end
|
19
|
+
|
20
|
+
def evalsha(sha, keys, argv)
|
21
|
+
@client.call("EVALSHA", sha, keys.size, *keys, *argv)
|
22
|
+
end
|
23
|
+
|
24
|
+
# this is the set of Redis commands used by Sidekiq. Not guaranteed
|
25
|
+
# to be comprehensive, we use this as a performance enhancement to
|
26
|
+
# avoid calling method_missing on most commands
|
27
|
+
USED_COMMANDS = %w[bitfield bitfield_ro del exists expire flushdb
|
28
|
+
get hdel hget hgetall hincrby hlen hmget hset hsetnx incr incrby
|
29
|
+
lindex llen lmove lpop lpush lrange lrem mget mset ping pttl
|
30
|
+
publish rpop rpush sadd scard script set sismember smembers
|
31
|
+
srem ttl type unlink zadd zcard zincrby zrange zrem
|
32
|
+
zremrangebyrank zremrangebyscore]
|
33
|
+
|
34
|
+
USED_COMMANDS.each do |name|
|
35
|
+
define_method(name) do |*args|
|
36
|
+
@client.call(name, *args)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
# this allows us to use methods like `conn.hmset(...)` instead of having to use
|
43
|
+
# redis-client's native `conn.call("hmset", ...)`
|
44
|
+
def method_missing(*args, &block)
|
45
|
+
warn("[sidekiq#5788] Redis has deprecated the `#{args.first}`command, called at #{caller(1..1)}") if DEPRECATED_COMMANDS.include?(args.first)
|
46
|
+
@client.call(*args, *block)
|
47
|
+
end
|
48
|
+
ruby2_keywords :method_missing if respond_to?(:ruby2_keywords, true)
|
49
|
+
|
50
|
+
def respond_to_missing?(name, include_private = false)
|
51
|
+
super # Appease the linter. We can't tell what is a valid command.
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
CompatClient = RedisClient::Decorator.create(CompatMethods)
|
56
|
+
|
57
|
+
class CompatClient
|
58
|
+
def config
|
59
|
+
@client.config
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def initialize(options)
|
64
|
+
opts = client_opts(options)
|
65
|
+
@config = if opts.key?(:sentinels)
|
66
|
+
RedisClient.sentinel(**opts)
|
67
|
+
else
|
68
|
+
RedisClient.config(**opts)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def new_client
|
73
|
+
CompatClient.new(@config.new_client)
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
def client_opts(options)
|
79
|
+
opts = options.dup
|
80
|
+
|
81
|
+
if opts[:namespace]
|
82
|
+
raise ArgumentError, "Your Redis configuration uses the namespace '#{opts[:namespace]}' but this feature is no longer supported in Sidekiq 7+. See https://github.com/sidekiq/sidekiq/blob/main/docs/7.0-Upgrade.md#redis-namespace."
|
83
|
+
end
|
84
|
+
|
85
|
+
opts.delete(:size)
|
86
|
+
opts.delete(:pool_timeout)
|
87
|
+
|
88
|
+
if opts[:network_timeout]
|
89
|
+
opts[:timeout] = opts[:network_timeout]
|
90
|
+
opts.delete(:network_timeout)
|
91
|
+
end
|
92
|
+
|
93
|
+
if opts[:driver]
|
94
|
+
opts[:driver] = opts[:driver].to_sym
|
95
|
+
end
|
96
|
+
|
97
|
+
opts[:name] = opts.delete(:master_name) if opts.key?(:master_name)
|
98
|
+
opts[:role] = opts[:role].to_sym if opts.key?(:role)
|
99
|
+
opts.delete(:url) if opts.key?(:sentinels)
|
100
|
+
|
101
|
+
# Issue #3303, redis-rb will silently retry an operation.
|
102
|
+
# This can lead to duplicate jobs if Sidekiq::Client's LPUSH
|
103
|
+
# is performed twice but I believe this is much, much rarer
|
104
|
+
# than the reconnect silently fixing a problem; we keep it
|
105
|
+
# on by default.
|
106
|
+
opts[:reconnect_attempts] ||= 1
|
107
|
+
|
108
|
+
opts
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|