sidekiq 7.3.8 → 8.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Changes.md +64 -0
- data/README.md +16 -13
- data/bin/sidekiqload +10 -10
- data/bin/webload +69 -0
- data/lib/active_job/queue_adapters/sidekiq_adapter.rb +90 -67
- data/lib/sidekiq/api.rb +122 -38
- data/lib/sidekiq/capsule.rb +6 -6
- data/lib/sidekiq/cli.rb +15 -19
- data/lib/sidekiq/client.rb +13 -16
- data/lib/sidekiq/component.rb +40 -2
- data/lib/sidekiq/config.rb +20 -16
- data/lib/sidekiq/embedded.rb +2 -1
- data/lib/sidekiq/iterable_job.rb +1 -0
- data/lib/sidekiq/job/iterable.rb +14 -5
- data/lib/sidekiq/job_logger.rb +4 -4
- data/lib/sidekiq/job_retry.rb +17 -5
- data/lib/sidekiq/job_util.rb +5 -1
- data/lib/sidekiq/launcher.rb +2 -1
- data/lib/sidekiq/logger.rb +19 -70
- data/lib/sidekiq/manager.rb +0 -1
- data/lib/sidekiq/metrics/query.rb +71 -45
- data/lib/sidekiq/metrics/shared.rb +8 -5
- data/lib/sidekiq/metrics/tracking.rb +9 -7
- data/lib/sidekiq/middleware/current_attributes.rb +5 -17
- data/lib/sidekiq/paginator.rb +8 -1
- data/lib/sidekiq/processor.rb +21 -14
- data/lib/sidekiq/profiler.rb +72 -0
- data/lib/sidekiq/rails.rb +43 -55
- data/lib/sidekiq/redis_client_adapter.rb +0 -1
- data/lib/sidekiq/redis_connection.rb +14 -3
- data/lib/sidekiq/testing.rb +2 -2
- data/lib/sidekiq/version.rb +2 -2
- data/lib/sidekiq/web/action.rb +122 -83
- data/lib/sidekiq/web/application.rb +345 -332
- data/lib/sidekiq/web/config.rb +117 -0
- data/lib/sidekiq/web/helpers.rb +41 -16
- data/lib/sidekiq/web/router.rb +60 -76
- data/lib/sidekiq/web.rb +50 -156
- data/lib/sidekiq.rb +2 -2
- data/sidekiq.gemspec +6 -6
- data/web/assets/javascripts/application.js +6 -13
- data/web/assets/javascripts/base-charts.js +30 -16
- data/web/assets/javascripts/chartjs-adapter-date-fns.min.js +7 -0
- data/web/assets/javascripts/dashboard-charts.js +2 -0
- data/web/assets/javascripts/dashboard.js +6 -0
- data/web/assets/javascripts/metrics.js +16 -34
- data/web/assets/stylesheets/style.css +757 -0
- data/web/locales/ar.yml +1 -0
- data/web/locales/cs.yml +1 -0
- data/web/locales/da.yml +1 -0
- data/web/locales/de.yml +1 -0
- data/web/locales/el.yml +1 -0
- data/web/locales/en.yml +9 -0
- data/web/locales/es.yml +24 -2
- data/web/locales/fa.yml +1 -0
- data/web/locales/fr.yml +1 -0
- data/web/locales/gd.yml +1 -0
- data/web/locales/he.yml +1 -0
- data/web/locales/hi.yml +1 -0
- data/web/locales/it.yml +8 -0
- data/web/locales/ja.yml +1 -0
- data/web/locales/ko.yml +1 -0
- data/web/locales/lt.yml +1 -0
- data/web/locales/nb.yml +1 -0
- data/web/locales/nl.yml +1 -0
- data/web/locales/pl.yml +1 -0
- data/web/locales/{pt-br.yml → pt-BR.yml} +2 -1
- data/web/locales/pt.yml +1 -0
- data/web/locales/ru.yml +1 -0
- data/web/locales/sv.yml +1 -0
- data/web/locales/ta.yml +1 -0
- data/web/locales/tr.yml +1 -0
- data/web/locales/uk.yml +1 -0
- data/web/locales/ur.yml +1 -0
- data/web/locales/vi.yml +1 -0
- data/web/locales/{zh-cn.yml → zh-CN.yml} +85 -73
- data/web/locales/{zh-tw.yml → zh-TW.yml} +2 -1
- data/web/views/_footer.erb +31 -34
- data/web/views/_job_info.erb +91 -89
- data/web/views/_metrics_period_select.erb +13 -10
- data/web/views/_nav.erb +14 -21
- data/web/views/_paging.erb +23 -21
- data/web/views/_poll_link.erb +2 -2
- data/web/views/_summary.erb +16 -16
- data/web/views/busy.erb +124 -122
- data/web/views/dashboard.erb +63 -64
- data/web/views/dead.erb +31 -27
- data/web/views/filtering.erb +3 -3
- data/web/views/layout.erb +13 -29
- data/web/views/metrics.erb +75 -81
- data/web/views/metrics_for_job.erb +45 -46
- data/web/views/morgue.erb +61 -70
- data/web/views/profiles.erb +43 -0
- data/web/views/queue.erb +54 -52
- data/web/views/queues.erb +43 -41
- data/web/views/retries.erb +66 -75
- data/web/views/retry.erb +32 -27
- data/web/views/scheduled.erb +58 -54
- data/web/views/scheduled_job_info.erb +1 -1
- metadata +25 -28
- data/web/assets/stylesheets/application-dark.css +0 -147
- data/web/assets/stylesheets/application-rtl.css +0 -163
- data/web/assets/stylesheets/application.css +0 -759
- data/web/assets/stylesheets/bootstrap-rtl.min.css +0 -9
- data/web/assets/stylesheets/bootstrap.css +0 -5
- data/web/views/_status.erb +0 -4
data/lib/sidekiq/api.rb
CHANGED
@@ -1,10 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "sidekiq"
|
4
|
-
|
5
3
|
require "zlib"
|
6
|
-
require "set"
|
7
4
|
|
5
|
+
require "sidekiq"
|
8
6
|
require "sidekiq/metrics/query"
|
9
7
|
|
10
8
|
#
|
@@ -101,11 +99,22 @@ module Sidekiq
|
|
101
99
|
rescue
|
102
100
|
{}
|
103
101
|
end
|
104
|
-
|
105
|
-
|
106
|
-
|
102
|
+
|
103
|
+
enqueued_at = job["enqueued_at"]
|
104
|
+
if enqueued_at
|
105
|
+
if enqueued_at.is_a?(Float)
|
106
|
+
# old format
|
107
|
+
now = Time.now.to_f
|
108
|
+
now - enqueued_at
|
109
|
+
else
|
110
|
+
now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
|
111
|
+
(now - enqueued_at) / 1000.0
|
112
|
+
end
|
113
|
+
else
|
114
|
+
0.0
|
115
|
+
end
|
107
116
|
else
|
108
|
-
0
|
117
|
+
0.0
|
109
118
|
end
|
110
119
|
|
111
120
|
@stats = {
|
@@ -265,11 +274,22 @@ module Sidekiq
|
|
265
274
|
entry = Sidekiq.redis { |conn|
|
266
275
|
conn.lindex(@rname, -1)
|
267
276
|
}
|
268
|
-
return 0 unless entry
|
277
|
+
return 0.0 unless entry
|
278
|
+
|
269
279
|
job = Sidekiq.load_json(entry)
|
270
|
-
|
271
|
-
|
272
|
-
|
280
|
+
enqueued_at = job["enqueued_at"]
|
281
|
+
if enqueued_at
|
282
|
+
if enqueued_at.is_a?(Float)
|
283
|
+
# old format
|
284
|
+
now = Time.now.to_f
|
285
|
+
now - enqueued_at
|
286
|
+
else
|
287
|
+
now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
|
288
|
+
(now - enqueued_at) / 1000.0
|
289
|
+
end
|
290
|
+
else
|
291
|
+
0.0
|
292
|
+
end
|
273
293
|
end
|
274
294
|
|
275
295
|
def each
|
@@ -421,12 +441,26 @@ module Sidekiq
|
|
421
441
|
self["bid"]
|
422
442
|
end
|
423
443
|
|
444
|
+
def failed_at
|
445
|
+
if self["failed_at"]
|
446
|
+
time_from_timestamp(self["failed_at"])
|
447
|
+
end
|
448
|
+
end
|
449
|
+
|
450
|
+
def retried_at
|
451
|
+
if self["retried_at"]
|
452
|
+
time_from_timestamp(self["retried_at"])
|
453
|
+
end
|
454
|
+
end
|
455
|
+
|
424
456
|
def enqueued_at
|
425
|
-
|
457
|
+
if self["enqueued_at"]
|
458
|
+
time_from_timestamp(self["enqueued_at"])
|
459
|
+
end
|
426
460
|
end
|
427
461
|
|
428
462
|
def created_at
|
429
|
-
|
463
|
+
time_from_timestamp(self["created_at"] || self["enqueued_at"] || 0)
|
430
464
|
end
|
431
465
|
|
432
466
|
def tags
|
@@ -444,8 +478,17 @@ module Sidekiq
|
|
444
478
|
end
|
445
479
|
|
446
480
|
def latency
|
447
|
-
|
448
|
-
|
481
|
+
timestamp = @item["enqueued_at"] || @item["created_at"]
|
482
|
+
if timestamp
|
483
|
+
if timestamp.is_a?(Float)
|
484
|
+
# old format
|
485
|
+
Time.now.to_f - timestamp
|
486
|
+
else
|
487
|
+
(::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) - timestamp) / 1000.0
|
488
|
+
end
|
489
|
+
else
|
490
|
+
0.0
|
491
|
+
end
|
449
492
|
end
|
450
493
|
|
451
494
|
# Remove this job from the queue
|
@@ -494,6 +537,15 @@ module Sidekiq
|
|
494
537
|
uncompressed = Zlib::Inflate.inflate(strict_base64_decoded)
|
495
538
|
Sidekiq.load_json(uncompressed)
|
496
539
|
end
|
540
|
+
|
541
|
+
def time_from_timestamp(timestamp)
|
542
|
+
if timestamp.is_a?(Float)
|
543
|
+
# old format, timestamps were stored as fractional seconds since the epoch
|
544
|
+
Time.at(timestamp).utc
|
545
|
+
else
|
546
|
+
Time.at(timestamp / 1000, timestamp % 1000, :millisecond)
|
547
|
+
end
|
548
|
+
end
|
497
549
|
end
|
498
550
|
|
499
551
|
# Represents a job within a Redis sorted set where the score
|
@@ -1114,8 +1166,8 @@ module Sidekiq
|
|
1114
1166
|
# works.each do |process_id, thread_id, work|
|
1115
1167
|
# # process_id is a unique identifier per Sidekiq process
|
1116
1168
|
# # thread_id is a unique identifier per thread
|
1117
|
-
# # work is a
|
1118
|
-
# #
|
1169
|
+
# # work is a `Sidekiq::Work` instance that has the following accessor methods.
|
1170
|
+
# # [work.queue, work.run_at, work.payload]
|
1119
1171
|
# # run_at is an epoch Integer.
|
1120
1172
|
# end
|
1121
1173
|
#
|
@@ -1142,7 +1194,7 @@ module Sidekiq
|
|
1142
1194
|
end
|
1143
1195
|
end
|
1144
1196
|
|
1145
|
-
results.sort_by { |(_, _,
|
1197
|
+
results.sort_by { |(_, _, work)| work.run_at }.each(&block)
|
1146
1198
|
end
|
1147
1199
|
|
1148
1200
|
# Note that #size is only as accurate as Sidekiq's heartbeat,
|
@@ -1172,13 +1224,14 @@ module Sidekiq
|
|
1172
1224
|
#
|
1173
1225
|
# @param jid [String] the job identifier
|
1174
1226
|
# @return [Sidekiq::Work] the work or nil
|
1175
|
-
def
|
1227
|
+
def find_work(jid)
|
1176
1228
|
each do |_process_id, _thread_id, work|
|
1177
1229
|
job = work.job
|
1178
1230
|
return work if job.jid == jid
|
1179
1231
|
end
|
1180
1232
|
nil
|
1181
1233
|
end
|
1234
|
+
alias_method :find_work_by_jid, :find_work
|
1182
1235
|
end
|
1183
1236
|
|
1184
1237
|
# Sidekiq::Work represents a job which is currently executing.
|
@@ -1208,33 +1261,64 @@ module Sidekiq
|
|
1208
1261
|
def payload
|
1209
1262
|
@hsh["payload"]
|
1210
1263
|
end
|
1264
|
+
end
|
1211
1265
|
|
1212
|
-
|
1213
|
-
|
1214
|
-
|
1215
|
-
|
1216
|
-
warn("Direct access to `Sidekiq::Work` attributes is deprecated, please use `#payload`, `#queue`, `#run_at` or `#job` instead", **kwargs)
|
1266
|
+
# Since "worker" is a nebulous term, we've deprecated the use of this class name.
|
1267
|
+
# Is "worker" a process, a type of job, a thread? Undefined!
|
1268
|
+
# WorkSet better describes the data.
|
1269
|
+
Workers = WorkSet
|
1217
1270
|
|
1218
|
-
|
1219
|
-
|
1271
|
+
class ProfileSet
|
1272
|
+
include Enumerable
|
1220
1273
|
|
1221
|
-
#
|
1222
|
-
#
|
1223
|
-
def
|
1224
|
-
@
|
1274
|
+
# This is a point in time/snapshot API, you'll need to instantiate a new instance
|
1275
|
+
# if you want to fetch newer records.
|
1276
|
+
def initialize
|
1277
|
+
@records = Sidekiq.redis do |c|
|
1278
|
+
# This throws away expired profiles
|
1279
|
+
c.zremrangebyscore("profiles", "-inf", Time.now.to_f.to_s)
|
1280
|
+
# retreive records, newest to oldest
|
1281
|
+
c.zrange("profiles", "+inf", 0, "byscore", "rev")
|
1282
|
+
end
|
1225
1283
|
end
|
1226
1284
|
|
1227
|
-
def
|
1228
|
-
@
|
1285
|
+
def size
|
1286
|
+
@records.size
|
1229
1287
|
end
|
1230
1288
|
|
1231
|
-
def
|
1232
|
-
|
1289
|
+
def each(&block)
|
1290
|
+
fetch_keys = %w[started_at jid type token size elapsed].freeze
|
1291
|
+
arrays = Sidekiq.redis do |c|
|
1292
|
+
c.pipelined do |p|
|
1293
|
+
@records.each do |key|
|
1294
|
+
p.hmget(key, *fetch_keys)
|
1295
|
+
end
|
1296
|
+
end
|
1297
|
+
end
|
1298
|
+
|
1299
|
+
arrays.compact.map { |arr| ProfileRecord.new(arr) }.each(&block)
|
1233
1300
|
end
|
1234
1301
|
end
|
1235
1302
|
|
1236
|
-
|
1237
|
-
|
1238
|
-
|
1239
|
-
|
1303
|
+
class ProfileRecord
|
1304
|
+
attr_reader :started_at, :jid, :type, :token, :size, :elapsed
|
1305
|
+
|
1306
|
+
def initialize(arr)
|
1307
|
+
# Must be same order as fetch_keys above
|
1308
|
+
@started_at = Time.at(Integer(arr[0]))
|
1309
|
+
@jid = arr[1]
|
1310
|
+
@type = arr[2]
|
1311
|
+
@token = arr[3]
|
1312
|
+
@size = Integer(arr[4])
|
1313
|
+
@elapsed = Float(arr[5])
|
1314
|
+
end
|
1315
|
+
|
1316
|
+
def key
|
1317
|
+
"#{token}-#{jid}"
|
1318
|
+
end
|
1319
|
+
|
1320
|
+
def data
|
1321
|
+
Sidekiq.redis { |c| c.hget(key, "data") }
|
1322
|
+
end
|
1323
|
+
end
|
1240
1324
|
end
|
data/lib/sidekiq/capsule.rb
CHANGED
@@ -11,12 +11,12 @@ module Sidekiq
|
|
11
11
|
# This capsule will pull jobs from the "single" queue and process
|
12
12
|
# the jobs with one thread, meaning the jobs will be processed serially.
|
13
13
|
#
|
14
|
-
#
|
15
|
-
#
|
16
|
-
#
|
17
|
-
#
|
14
|
+
# Sidekiq.configure_server do |config|
|
15
|
+
# config.capsule("single-threaded") do |cap|
|
16
|
+
# cap.concurrency = 1
|
17
|
+
# cap.queues = %w(single)
|
18
|
+
# end
|
18
19
|
# end
|
19
|
-
# end
|
20
20
|
class Capsule
|
21
21
|
include Sidekiq::Component
|
22
22
|
extend Forwardable
|
@@ -27,7 +27,7 @@ module Sidekiq
|
|
27
27
|
attr_reader :mode
|
28
28
|
attr_reader :weights
|
29
29
|
|
30
|
-
def_delegators :@config, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
|
30
|
+
def_delegators :@config, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig, :thread_priority
|
31
31
|
|
32
32
|
def initialize(name, config)
|
33
33
|
@name = name
|
data/lib/sidekiq/cli.rb
CHANGED
@@ -3,7 +3,6 @@
|
|
3
3
|
$stdout.sync = true
|
4
4
|
|
5
5
|
require "yaml"
|
6
|
-
require "singleton"
|
7
6
|
require "optparse"
|
8
7
|
require "erb"
|
9
8
|
require "fileutils"
|
@@ -17,7 +16,6 @@ require "sidekiq/launcher"
|
|
17
16
|
module Sidekiq # :nodoc:
|
18
17
|
class CLI
|
19
18
|
include Sidekiq::Component
|
20
|
-
include Singleton unless $TESTING
|
21
19
|
|
22
20
|
attr_accessor :launcher
|
23
21
|
attr_accessor :environment
|
@@ -31,6 +29,10 @@ module Sidekiq # :nodoc:
|
|
31
29
|
validate!
|
32
30
|
end
|
33
31
|
|
32
|
+
def self.instance
|
33
|
+
@instance ||= new
|
34
|
+
end
|
35
|
+
|
34
36
|
def jruby?
|
35
37
|
defined?(::JRUBY_VERSION)
|
36
38
|
end
|
@@ -74,7 +76,7 @@ module Sidekiq # :nodoc:
|
|
74
76
|
# fire startup and start multithreading.
|
75
77
|
info = @config.redis_info
|
76
78
|
ver = Gem::Version.new(info["redis_version"])
|
77
|
-
raise "You are
|
79
|
+
raise "You are connected to Redis #{ver}, Sidekiq requires Redis 7.0.0 or greater" if ver < Gem::Version.new("7.0.0")
|
78
80
|
|
79
81
|
maxmemory_policy = info["maxmemory_policy"]
|
80
82
|
if maxmemory_policy != "noeviction" && maxmemory_policy != ""
|
@@ -298,29 +300,18 @@ module Sidekiq # :nodoc:
|
|
298
300
|
|
299
301
|
if File.directory?(@config[:require])
|
300
302
|
require "rails"
|
301
|
-
if ::Rails::VERSION::MAJOR <
|
302
|
-
warn "Sidekiq #{Sidekiq::VERSION} only supports Rails
|
303
|
+
if ::Rails::VERSION::MAJOR < 7
|
304
|
+
warn "Sidekiq #{Sidekiq::VERSION} only supports Rails 7+"
|
303
305
|
end
|
304
306
|
require "sidekiq/rails"
|
305
307
|
require File.expand_path("#{@config[:require]}/config/environment.rb")
|
306
|
-
@config[:tag] ||= default_tag
|
308
|
+
@config[:tag] ||= default_tag(::Rails.root)
|
307
309
|
else
|
308
310
|
require @config[:require]
|
311
|
+
@config[:tag] ||= default_tag
|
309
312
|
end
|
310
313
|
end
|
311
314
|
|
312
|
-
def default_tag
|
313
|
-
dir = ::Rails.root
|
314
|
-
name = File.basename(dir)
|
315
|
-
prevdir = File.dirname(dir) # Capistrano release directory?
|
316
|
-
if name.to_i != 0 && prevdir
|
317
|
-
if File.basename(prevdir) == "releases"
|
318
|
-
return File.basename(File.dirname(prevdir))
|
319
|
-
end
|
320
|
-
end
|
321
|
-
name
|
322
|
-
end
|
323
|
-
|
324
315
|
def validate!
|
325
316
|
if !File.exist?(@config[:require]) ||
|
326
317
|
(File.directory?(@config[:require]) && !File.exist?("#{@config[:require]}/config/application.rb"))
|
@@ -395,7 +386,12 @@ module Sidekiq # :nodoc:
|
|
395
386
|
end
|
396
387
|
|
397
388
|
def initialize_logger
|
398
|
-
|
389
|
+
if @config[:verbose] || ENV["DEBUG_INVOCATION"] == "1"
|
390
|
+
# DEBUG_INVOCATION is a systemd-ism triggered by
|
391
|
+
# RestartMode=debug. We turn on debugging when the
|
392
|
+
# sidekiq process crashes and is restarted with this flag.
|
393
|
+
@config.logger.level = ::Logger::DEBUG
|
394
|
+
end
|
399
395
|
end
|
400
396
|
|
401
397
|
def parse_config(path)
|
data/lib/sidekiq/client.rb
CHANGED
@@ -67,9 +67,7 @@ module Sidekiq
|
|
67
67
|
c.pipelined do |p|
|
68
68
|
p.hsetnx(key, "cancelled", Time.now.to_i)
|
69
69
|
p.hget(key, "cancelled")
|
70
|
-
p.expire(key, Sidekiq::Job::Iterable::STATE_TTL)
|
71
|
-
# TODO When Redis 7.2 is required
|
72
|
-
# p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
|
70
|
+
p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
|
73
71
|
end
|
74
72
|
end
|
75
73
|
result.to_i
|
@@ -266,22 +264,21 @@ module Sidekiq
|
|
266
264
|
if payloads.first.key?("at")
|
267
265
|
conn.zadd("schedule", payloads.flat_map { |hash|
|
268
266
|
at = hash["at"].to_s
|
269
|
-
# ActiveJob sets
|
270
|
-
hash.
|
271
|
-
# TODO: Use hash.except("at") when support for Ruby 2.7 is dropped
|
272
|
-
hash = hash.dup
|
273
|
-
hash.delete("at")
|
267
|
+
# ActiveJob sets enqueued_at but the job has not been enqueued yet
|
268
|
+
hash = hash.except("enqueued_at", "at")
|
274
269
|
[at, Sidekiq.dump_json(hash)]
|
275
270
|
})
|
276
271
|
else
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
272
|
+
now = ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond) # milliseconds since the epoch
|
273
|
+
grouped_queues = payloads.group_by { |job| job["queue"] }
|
274
|
+
conn.sadd("queues", grouped_queues.keys)
|
275
|
+
grouped_queues.each do |queue, grouped_payloads|
|
276
|
+
to_push = grouped_payloads.map { |entry|
|
277
|
+
entry["enqueued_at"] = now
|
278
|
+
Sidekiq.dump_json(entry)
|
279
|
+
}
|
280
|
+
conn.lpush("queue:#{queue}", to_push)
|
281
|
+
end
|
285
282
|
end
|
286
283
|
end
|
287
284
|
end
|
data/lib/sidekiq/component.rb
CHANGED
@@ -1,11 +1,38 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module Sidekiq
|
4
|
+
# Ruby's default thread priority is 0, which uses 100ms time slices.
|
5
|
+
# This can lead to some surprising thread starvation; if using a lot of
|
6
|
+
# CPU-heavy concurrency, it may take several seconds before a Thread gets
|
7
|
+
# on the CPU.
|
8
|
+
#
|
9
|
+
# Negative priorities lower the timeslice by half, so -1 = 50ms, -2 = 25ms, etc.
|
10
|
+
# With more frequent timeslices, we reduce the risk of unintentional timeouts
|
11
|
+
# and starvation.
|
12
|
+
#
|
13
|
+
# Customize like so:
|
14
|
+
#
|
15
|
+
# Sidekiq.configure_server do |cfg|
|
16
|
+
# cfg.thread_priority = 0
|
17
|
+
# end
|
18
|
+
#
|
19
|
+
DEFAULT_THREAD_PRIORITY = -1
|
20
|
+
|
4
21
|
##
|
5
22
|
# Sidekiq::Component assumes a config instance is available at @config
|
6
23
|
module Component # :nodoc:
|
7
24
|
attr_reader :config
|
8
25
|
|
26
|
+
# This is epoch milliseconds, appropriate for persistence
|
27
|
+
def real_ms
|
28
|
+
::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
|
29
|
+
end
|
30
|
+
|
31
|
+
# used for time difference and relative comparisons, not persistence.
|
32
|
+
def mono_ms
|
33
|
+
::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
|
34
|
+
end
|
35
|
+
|
9
36
|
def watchdog(last_words)
|
10
37
|
yield
|
11
38
|
rescue Exception => ex
|
@@ -13,11 +40,11 @@ module Sidekiq
|
|
13
40
|
raise ex
|
14
41
|
end
|
15
42
|
|
16
|
-
def safe_thread(name, &block)
|
43
|
+
def safe_thread(name, priority: nil, &block)
|
17
44
|
Thread.new do
|
18
45
|
Thread.current.name = "sidekiq.#{name}"
|
19
46
|
watchdog(name, &block)
|
20
|
-
end
|
47
|
+
end.tap { |t| t.priority = (priority || config.thread_priority || DEFAULT_THREAD_PRIORITY) }
|
21
48
|
end
|
22
49
|
|
23
50
|
def logger
|
@@ -86,5 +113,16 @@ module Sidekiq
|
|
86
113
|
end.join(", ")
|
87
114
|
}>"
|
88
115
|
end
|
116
|
+
|
117
|
+
def default_tag(dir = Dir.pwd)
|
118
|
+
name = File.basename(dir)
|
119
|
+
prevdir = File.dirname(dir) # Capistrano release directory?
|
120
|
+
if name.to_i != 0 && prevdir
|
121
|
+
if File.basename(prevdir) == "releases"
|
122
|
+
return File.basename(File.dirname(prevdir))
|
123
|
+
end
|
124
|
+
end
|
125
|
+
name
|
126
|
+
end
|
89
127
|
end
|
90
128
|
end
|
data/lib/sidekiq/config.rb
CHANGED
@@ -1,8 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "forwardable"
|
4
|
-
|
5
|
-
require "set"
|
6
4
|
require "sidekiq/redis_connection"
|
7
5
|
|
8
6
|
module Sidekiq
|
@@ -29,6 +27,7 @@ module Sidekiq
|
|
29
27
|
startup: [],
|
30
28
|
quiet: [],
|
31
29
|
shutdown: [],
|
30
|
+
exit: [],
|
32
31
|
# triggers when we fire the first heartbeat on startup OR repairing a network partition
|
33
32
|
heartbeat: [],
|
34
33
|
# triggers on EVERY heartbeat call, every 10 seconds
|
@@ -41,12 +40,22 @@ module Sidekiq
|
|
41
40
|
}
|
42
41
|
|
43
42
|
ERROR_HANDLER = ->(ex, ctx, cfg = Sidekiq.default_configuration) {
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
43
|
+
Sidekiq::Context.with(ctx) do
|
44
|
+
dev = cfg[:environment] == "development"
|
45
|
+
fancy = dev && $stdout.tty? # 🎩
|
46
|
+
# Weird logic here but we want to show the backtrace in local
|
47
|
+
# development or if verbose logging is enabled.
|
48
|
+
#
|
49
|
+
# `full_message` contains the error class, message and backtrace
|
50
|
+
# `detailed_message` contains the error class and message
|
51
|
+
#
|
52
|
+
# Absolutely terrible API names. Not useful at all to have two
|
53
|
+
# methods with similar but obscure names.
|
54
|
+
if dev || cfg.logger.debug?
|
55
|
+
cfg.logger.info { ex.full_message(highlight: fancy) }
|
56
|
+
else
|
57
|
+
cfg.logger.info { ex.detailed_message(highlight: fancy) }
|
58
|
+
end
|
50
59
|
end
|
51
60
|
}
|
52
61
|
|
@@ -60,6 +69,7 @@ module Sidekiq
|
|
60
69
|
|
61
70
|
def_delegators :@options, :[], :[]=, :fetch, :key?, :has_key?, :merge!, :dig
|
62
71
|
attr_reader :capsules
|
72
|
+
attr_accessor :thread_priority
|
63
73
|
|
64
74
|
def inspect
|
65
75
|
"#<#{self.class.name} @options=#{
|
@@ -249,7 +259,7 @@ module Sidekiq
|
|
249
259
|
end
|
250
260
|
|
251
261
|
# Register a block to run at a point in the Sidekiq lifecycle.
|
252
|
-
# :startup, :quiet or :
|
262
|
+
# :startup, :quiet, :shutdown, or :exit are valid events.
|
253
263
|
#
|
254
264
|
# Sidekiq.configure_server do |config|
|
255
265
|
# config.on(:shutdown) do
|
@@ -293,13 +303,7 @@ module Sidekiq
|
|
293
303
|
p ["!!!!!", ex]
|
294
304
|
end
|
295
305
|
@options[:error_handlers].each do |handler|
|
296
|
-
|
297
|
-
# TODO Remove in 8.0
|
298
|
-
logger.info { "DEPRECATION: Sidekiq exception handlers now take three arguments, see #{handler}" }
|
299
|
-
handler.call(ex, {_config: self}.merge(ctx))
|
300
|
-
else
|
301
|
-
handler.call(ex, ctx, self)
|
302
|
-
end
|
306
|
+
handler.call(ex, ctx, self)
|
303
307
|
rescue Exception => e
|
304
308
|
l = logger
|
305
309
|
l.error "!!! ERROR HANDLER THREW AN ERROR !!!"
|
data/lib/sidekiq/embedded.rb
CHANGED
@@ -34,6 +34,7 @@ module Sidekiq
|
|
34
34
|
private
|
35
35
|
|
36
36
|
def housekeeping
|
37
|
+
@config[:tag] ||= default_tag
|
37
38
|
logger.info "Running in #{RUBY_DESCRIPTION}"
|
38
39
|
logger.info Sidekiq::LICENSE
|
39
40
|
logger.info "Upgrade to Sidekiq Pro for more features and support: https://sidekiq.org" unless defined?(::Sidekiq::Pro)
|
@@ -42,7 +43,7 @@ module Sidekiq
|
|
42
43
|
# fire startup and start multithreading.
|
43
44
|
info = config.redis_info
|
44
45
|
ver = Gem::Version.new(info["redis_version"])
|
45
|
-
raise "You are
|
46
|
+
raise "You are connected to Redis #{ver}, Sidekiq requires Redis 7.0.0 or greater" if ver < Gem::Version.new("7.0.0")
|
46
47
|
|
47
48
|
maxmemory_policy = info["maxmemory_policy"]
|
48
49
|
if maxmemory_policy != "noeviction"
|
data/lib/sidekiq/iterable_job.rb
CHANGED
data/lib/sidekiq/job/iterable.rb
CHANGED
@@ -54,9 +54,7 @@ module Sidekiq
|
|
54
54
|
c.pipelined do |p|
|
55
55
|
p.hsetnx(key, "cancelled", Time.now.to_i)
|
56
56
|
p.hget(key, "cancelled")
|
57
|
-
|
58
|
-
# p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
|
59
|
-
p.expire(key, Sidekiq::Job::Iterable::STATE_TTL)
|
57
|
+
p.expire(key, Sidekiq::Job::Iterable::STATE_TTL, "nx")
|
60
58
|
end
|
61
59
|
end
|
62
60
|
@_cancelled = result.to_i
|
@@ -66,6 +64,10 @@ module Sidekiq
|
|
66
64
|
@_cancelled
|
67
65
|
end
|
68
66
|
|
67
|
+
def cursor
|
68
|
+
@_cursor.freeze
|
69
|
+
end
|
70
|
+
|
69
71
|
# A hook to override that will be called when the job starts iterating.
|
70
72
|
#
|
71
73
|
# It is called only once, for the first time.
|
@@ -93,6 +95,11 @@ module Sidekiq
|
|
93
95
|
def on_stop
|
94
96
|
end
|
95
97
|
|
98
|
+
# A hook to override that will be called when the job is cancelled.
|
99
|
+
#
|
100
|
+
def on_cancel
|
101
|
+
end
|
102
|
+
|
96
103
|
# A hook to override that will be called when the job finished iterating.
|
97
104
|
#
|
98
105
|
def on_complete
|
@@ -184,6 +191,7 @@ module Sidekiq
|
|
184
191
|
|
185
192
|
def iterate_with_enumerator(enumerator, arguments)
|
186
193
|
if is_cancelled?
|
194
|
+
on_cancel
|
187
195
|
logger.info { "Job cancelled" }
|
188
196
|
return true
|
189
197
|
end
|
@@ -200,8 +208,9 @@ module Sidekiq
|
|
200
208
|
if ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - state_flushed_at >= STATE_FLUSH_INTERVAL || is_interrupted
|
201
209
|
_, _, cancelled = flush_state
|
202
210
|
state_flushed_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
203
|
-
if cancelled
|
211
|
+
if cancelled
|
204
212
|
@_cancelled = true
|
213
|
+
on_cancel
|
205
214
|
logger.info { "Job cancelled" }
|
206
215
|
return true
|
207
216
|
end
|
@@ -265,7 +274,7 @@ module Sidekiq
|
|
265
274
|
Sidekiq.redis do |conn|
|
266
275
|
conn.multi do |pipe|
|
267
276
|
pipe.hset(key, state)
|
268
|
-
pipe.expire(key, STATE_TTL)
|
277
|
+
pipe.expire(key, STATE_TTL, "nx")
|
269
278
|
pipe.hget(key, "cancelled")
|
270
279
|
end
|
271
280
|
end
|
data/lib/sidekiq/job_logger.rb
CHANGED
@@ -26,16 +26,16 @@ module Sidekiq
|
|
26
26
|
# If we're using a wrapper class, like ActiveJob, use the "wrapped"
|
27
27
|
# attribute to expose the underlying thing.
|
28
28
|
h = {
|
29
|
-
|
30
|
-
|
29
|
+
jid: job_hash["jid"],
|
30
|
+
class: job_hash["display_class"] || job_hash["wrapped"] || job_hash["class"]
|
31
31
|
}
|
32
32
|
h[:bid] = job_hash["bid"] if job_hash.has_key?("bid")
|
33
33
|
h[:tags] = job_hash["tags"] if job_hash.has_key?("tags")
|
34
34
|
|
35
35
|
Thread.current[:sidekiq_context] = h
|
36
36
|
level = job_hash["log_level"]
|
37
|
-
if level
|
38
|
-
@logger.
|
37
|
+
if level
|
38
|
+
@logger.with_level(level, &block)
|
39
39
|
else
|
40
40
|
yield
|
41
41
|
end
|