sidekiq 7.3.9 → 8.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. checksums.yaml +4 -4
  2. data/Changes.md +57 -0
  3. data/README.md +16 -13
  4. data/bin/sidekiqload +10 -10
  5. data/bin/webload +69 -0
  6. data/lib/active_job/queue_adapters/sidekiq_adapter.rb +93 -57
  7. data/lib/sidekiq/api.rb +122 -38
  8. data/lib/sidekiq/capsule.rb +6 -6
  9. data/lib/sidekiq/cli.rb +15 -19
  10. data/lib/sidekiq/client.rb +13 -16
  11. data/lib/sidekiq/component.rb +40 -2
  12. data/lib/sidekiq/config.rb +20 -16
  13. data/lib/sidekiq/embedded.rb +2 -1
  14. data/lib/sidekiq/iterable_job.rb +1 -0
  15. data/lib/sidekiq/job/iterable.rb +13 -4
  16. data/lib/sidekiq/job_logger.rb +4 -4
  17. data/lib/sidekiq/job_retry.rb +17 -5
  18. data/lib/sidekiq/job_util.rb +5 -1
  19. data/lib/sidekiq/launcher.rb +2 -1
  20. data/lib/sidekiq/logger.rb +19 -70
  21. data/lib/sidekiq/manager.rb +0 -1
  22. data/lib/sidekiq/metrics/query.rb +71 -45
  23. data/lib/sidekiq/metrics/shared.rb +8 -5
  24. data/lib/sidekiq/metrics/tracking.rb +9 -7
  25. data/lib/sidekiq/middleware/current_attributes.rb +5 -17
  26. data/lib/sidekiq/paginator.rb +8 -1
  27. data/lib/sidekiq/processor.rb +21 -14
  28. data/lib/sidekiq/profiler.rb +72 -0
  29. data/lib/sidekiq/rails.rb +43 -65
  30. data/lib/sidekiq/redis_client_adapter.rb +0 -1
  31. data/lib/sidekiq/redis_connection.rb +14 -3
  32. data/lib/sidekiq/testing.rb +2 -2
  33. data/lib/sidekiq/version.rb +2 -2
  34. data/lib/sidekiq/web/action.rb +122 -83
  35. data/lib/sidekiq/web/application.rb +345 -332
  36. data/lib/sidekiq/web/config.rb +117 -0
  37. data/lib/sidekiq/web/helpers.rb +41 -16
  38. data/lib/sidekiq/web/router.rb +60 -76
  39. data/lib/sidekiq/web.rb +50 -156
  40. data/lib/sidekiq.rb +2 -2
  41. data/sidekiq.gemspec +6 -6
  42. data/web/assets/javascripts/application.js +6 -13
  43. data/web/assets/javascripts/base-charts.js +30 -16
  44. data/web/assets/javascripts/chartjs-adapter-date-fns.min.js +7 -0
  45. data/web/assets/javascripts/metrics.js +16 -34
  46. data/web/assets/stylesheets/style.css +757 -0
  47. data/web/locales/ar.yml +1 -0
  48. data/web/locales/cs.yml +1 -0
  49. data/web/locales/da.yml +1 -0
  50. data/web/locales/de.yml +1 -0
  51. data/web/locales/el.yml +1 -0
  52. data/web/locales/en.yml +6 -0
  53. data/web/locales/es.yml +24 -2
  54. data/web/locales/fa.yml +1 -0
  55. data/web/locales/fr.yml +1 -0
  56. data/web/locales/gd.yml +1 -0
  57. data/web/locales/he.yml +1 -0
  58. data/web/locales/hi.yml +1 -0
  59. data/web/locales/it.yml +8 -0
  60. data/web/locales/ja.yml +1 -0
  61. data/web/locales/ko.yml +1 -0
  62. data/web/locales/lt.yml +1 -0
  63. data/web/locales/nb.yml +1 -0
  64. data/web/locales/nl.yml +1 -0
  65. data/web/locales/pl.yml +1 -0
  66. data/web/locales/{pt-br.yml → pt-BR.yml} +2 -1
  67. data/web/locales/pt.yml +1 -0
  68. data/web/locales/ru.yml +1 -0
  69. data/web/locales/sv.yml +1 -0
  70. data/web/locales/ta.yml +1 -0
  71. data/web/locales/tr.yml +1 -0
  72. data/web/locales/uk.yml +1 -0
  73. data/web/locales/ur.yml +1 -0
  74. data/web/locales/vi.yml +1 -0
  75. data/web/locales/{zh-cn.yml → zh-CN.yml} +85 -73
  76. data/web/locales/{zh-tw.yml → zh-TW.yml} +2 -1
  77. data/web/views/_footer.erb +31 -33
  78. data/web/views/_job_info.erb +91 -89
  79. data/web/views/_metrics_period_select.erb +13 -10
  80. data/web/views/_nav.erb +14 -21
  81. data/web/views/_paging.erb +23 -21
  82. data/web/views/_poll_link.erb +2 -2
  83. data/web/views/_summary.erb +16 -16
  84. data/web/views/busy.erb +124 -122
  85. data/web/views/dashboard.erb +62 -66
  86. data/web/views/dead.erb +31 -27
  87. data/web/views/filtering.erb +3 -3
  88. data/web/views/layout.erb +13 -29
  89. data/web/views/metrics.erb +75 -81
  90. data/web/views/metrics_for_job.erb +45 -46
  91. data/web/views/morgue.erb +61 -70
  92. data/web/views/profiles.erb +43 -0
  93. data/web/views/queue.erb +54 -52
  94. data/web/views/queues.erb +43 -41
  95. data/web/views/retries.erb +66 -75
  96. data/web/views/retry.erb +32 -27
  97. data/web/views/scheduled.erb +58 -54
  98. data/web/views/scheduled_job_info.erb +1 -1
  99. metadata +24 -24
  100. data/web/assets/stylesheets/application-dark.css +0 -147
  101. data/web/assets/stylesheets/application-rtl.css +0 -163
  102. data/web/assets/stylesheets/application.css +0 -759
  103. data/web/assets/stylesheets/bootstrap-rtl.min.css +0 -9
  104. data/web/assets/stylesheets/bootstrap.css +0 -5
  105. data/web/views/_status.erb +0 -4
@@ -139,6 +139,10 @@ module Sidekiq
139
139
 
140
140
  private
141
141
 
142
+ def now_ms
143
+ ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
144
+ end
145
+
142
146
  # Note that +jobinst+ can be nil here if an error is raised before we can
143
147
  # instantiate the job instance. All access must be guarded and
144
148
  # best effort.
@@ -149,17 +153,17 @@ module Sidekiq
149
153
 
150
154
  m = exception_message(exception)
151
155
  if m.respond_to?(:scrub!)
152
- m.force_encoding("utf-8")
156
+ m.force_encoding(Encoding::UTF_8)
153
157
  m.scrub!
154
158
  end
155
159
 
156
160
  msg["error_message"] = m
157
161
  msg["error_class"] = exception.class.name
158
162
  count = if msg["retry_count"]
159
- msg["retried_at"] = Time.now.to_f
163
+ msg["retried_at"] = now_ms
160
164
  msg["retry_count"] += 1
161
165
  else
162
- msg["failed_at"] = Time.now.to_f
166
+ msg["failed_at"] = now_ms
163
167
  msg["retry_count"] = 0
164
168
  end
165
169
 
@@ -177,7 +181,7 @@ module Sidekiq
177
181
  return retries_exhausted(jobinst, msg, exception) if count >= max_retry_attempts
178
182
 
179
183
  rf = msg["retry_for"]
180
- return retries_exhausted(jobinst, msg, exception) if rf && ((msg["failed_at"] + rf) < Time.now.to_f)
184
+ return retries_exhausted(jobinst, msg, exception) if rf && (time_for(msg["failed_at"]) + rf) < Time.now
181
185
 
182
186
  strategy, delay = delay_for(jobinst, count, exception, msg)
183
187
  case strategy
@@ -189,7 +193,7 @@ module Sidekiq
189
193
 
190
194
  # Logging here can break retries if the logging device raises ENOSPC #3979
191
195
  # logger.debug { "Failure! Retry #{count} in #{delay} seconds" }
192
- jitter = rand(10) * (count + 1)
196
+ jitter = rand(10 * (count + 1))
193
197
  retry_at = Time.now.to_f + delay + jitter
194
198
  payload = Sidekiq.dump_json(msg)
195
199
  redis do |conn|
@@ -197,6 +201,14 @@ module Sidekiq
197
201
  end
198
202
  end
199
203
 
204
+ def time_for(item)
205
+ if item.is_a?(Float)
206
+ Time.at(item)
207
+ else
208
+ Time.at(item / 1000, item % 1000)
209
+ end
210
+ end
211
+
200
212
  # returns (strategy, seconds)
201
213
  def delay_for(jobinst, count, exception, msg)
202
214
  rv = begin
@@ -58,10 +58,14 @@ module Sidekiq
58
58
  item["class"] = item["class"].to_s
59
59
  item["queue"] = item["queue"].to_s
60
60
  item["retry_for"] = item["retry_for"].to_i if item["retry_for"]
61
- item["created_at"] ||= Time.now.to_f
61
+ item["created_at"] ||= now_in_millis
62
62
  item
63
63
  end
64
64
 
65
+ def now_in_millis
66
+ ::Process.clock_gettime(::Process::CLOCK_REALTIME, :millisecond)
67
+ end
68
+
65
69
  def normalized_hash(item_class)
66
70
  if item_class.is_a?(Class)
67
71
  raise(ArgumentError, "Message must include a Sidekiq::Job class, not class name: #{item_class.ancestors.inspect}") unless item_class.respond_to?(:get_sidekiq_options)
@@ -68,6 +68,7 @@ module Sidekiq
68
68
  stoppers.each(&:join)
69
69
 
70
70
  clear_heartbeat
71
+ fire_event(:exit, reverse: true)
71
72
  end
72
73
 
73
74
  def stopping?
@@ -81,7 +82,7 @@ module Sidekiq
81
82
 
82
83
  end
83
84
 
84
- private unless $TESTING
85
+ private
85
86
 
86
87
  BEAT_PAUSE = 10
87
88
 
@@ -22,92 +22,41 @@ module Sidekiq
22
22
  end
23
23
  end
24
24
 
25
- module LoggingUtils
26
- LEVELS = {
27
- "debug" => 0,
28
- "info" => 1,
29
- "warn" => 2,
30
- "error" => 3,
31
- "fatal" => 4
32
- }
33
- LEVELS.default_proc = proc do |_, level|
34
- puts("Invalid log level: #{level.inspect}")
35
- nil
36
- end
37
-
38
- LEVELS.each do |level, numeric_level|
39
- define_method(:"#{level}?") do
40
- local_level.nil? ? super() : local_level <= numeric_level
41
- end
42
- end
43
-
44
- def local_level
45
- Thread.current[:sidekiq_log_level]
46
- end
47
-
48
- def local_level=(level)
49
- case level
50
- when Integer
51
- Thread.current[:sidekiq_log_level] = level
52
- when Symbol, String
53
- Thread.current[:sidekiq_log_level] = LEVELS[level.to_s]
54
- when nil
55
- Thread.current[:sidekiq_log_level] = nil
56
- else
57
- raise ArgumentError, "Invalid log level: #{level.inspect}"
58
- end
59
- end
60
-
61
- def level
62
- local_level || super
63
- end
64
-
65
- # Change the thread-local level for the duration of the given block.
66
- def log_at(level)
67
- old_local_level = local_level
68
- self.local_level = level
69
- yield
70
- ensure
71
- self.local_level = old_local_level
72
- end
73
- end
74
-
75
25
  class Logger < ::Logger
76
- include LoggingUtils
77
-
78
26
  module Formatters
27
+ COLORS = {
28
+ "DEBUG" => "\e[1;32mDEBUG\e[0m", # green
29
+ "INFO" => "\e[1;34mINFO \e[0m", # blue
30
+ "WARN" => "\e[1;33mWARN \e[0m", # yellow
31
+ "ERROR" => "\e[1;31mERROR\e[0m", # red
32
+ "FATAL" => "\e[1;35mFATAL\e[0m" # pink
33
+ }
79
34
  class Base < ::Logger::Formatter
80
35
  def tid
81
36
  Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
82
37
  end
83
38
 
84
- def ctx
85
- Sidekiq::Context.current
86
- end
87
-
88
- def format_context
89
- if ctx.any?
90
- " " + ctx.compact.map { |k, v|
91
- case v
92
- when Array
93
- "#{k}=#{v.join(",")}"
94
- else
95
- "#{k}=#{v}"
96
- end
97
- }.join(" ")
98
- end
39
+ def format_context(ctxt = Sidekiq::Context.current)
40
+ (ctxt.size == 0) ? "" : " #{ctxt.map { |k, v|
41
+ case v
42
+ when Array
43
+ "#{k}=#{v.join(",")}"
44
+ else
45
+ "#{k}=#{v}"
46
+ end
47
+ }.join(" ")}"
99
48
  end
100
49
  end
101
50
 
102
51
  class Pretty < Base
103
52
  def call(severity, time, program_name, message)
104
- "#{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
53
+ "#{Formatters::COLORS[severity]} #{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid}#{format_context}: #{message}\n"
105
54
  end
106
55
  end
107
56
 
108
57
  class WithoutTimestamp < Pretty
109
58
  def call(severity, time, program_name, message)
110
- "pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
59
+ "#{Formatters::COLORS[severity]} pid=#{::Process.pid} tid=#{tid} #{format_context}: #{message}\n"
111
60
  end
112
61
  end
113
62
 
@@ -120,7 +69,7 @@ module Sidekiq
120
69
  lvl: severity,
121
70
  msg: message
122
71
  }
123
- c = ctx
72
+ c = Sidekiq::Context.current
124
73
  hash["ctx"] = c unless c.empty?
125
74
 
126
75
  Sidekiq.dump_json(hash) << "\n"
@@ -1,7 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "sidekiq/processor"
4
- require "set"
5
4
 
6
5
  module Sidekiq
7
6
  ##
@@ -1,9 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "sidekiq"
4
3
  require "date"
5
- require "set"
6
-
4
+ require "sidekiq"
7
5
  require "sidekiq/metrics/shared"
8
6
 
9
7
  module Sidekiq
@@ -12,7 +10,7 @@ module Sidekiq
12
10
  # Caller sets a set of attributes to act as filters. {#fetch} will call
13
11
  # Redis and return a Hash of results.
14
12
  #
15
- # NB: all metrics and times/dates are UTC only. We specifically do not
13
+ # NB: all metrics and times/dates are UTC only. We explicitly do not
16
14
  # support timezones.
17
15
  class Query
18
16
  def initialize(pool: nil, now: Time.now)
@@ -21,23 +19,46 @@ module Sidekiq
21
19
  @klass = nil
22
20
  end
23
21
 
22
+ ROLLUPS = {
23
+ # minutely aggregates per minute
24
+ minutely: [60, ->(time) { time.strftime("j|%y%m%d|%-H:%M") }],
25
+ # hourly aggregates every 10 minutes so we'll have six data points per hour
26
+ hourly: [600, ->(time) {
27
+ m = time.min
28
+ mins = (m < 10) ? "0" : m.to_s[0]
29
+ time.strftime("j|%y%m%d|%-H:#{mins}")
30
+ }]
31
+ }
32
+
24
33
  # Get metric data for all jobs from the last hour
25
34
  # +class_filter+: return only results for classes matching filter
26
- def top_jobs(class_filter: nil, minutes: 60)
27
- result = Result.new
28
-
35
+ # +minutes+: the number of fine-grained minute buckets to retrieve
36
+ # +hours+: the number of coarser-grained 10-minute buckets to retrieve, in hours
37
+ def top_jobs(class_filter: nil, minutes: nil, hours: nil)
29
38
  time = @time
39
+ minutes = 60 unless minutes || hours
40
+
41
+ # DoS protection, sanity check
42
+ minutes = 60 if minutes && minutes > 480
43
+ hours = 72 if hours && hours > 72
44
+
45
+ granularity = hours ? :hourly : :minutely
46
+ result = Result.new(granularity)
47
+ result.ends_at = time
48
+ count = hours ? hours * 6 : minutes
49
+ stride, keyproc = ROLLUPS[granularity]
50
+
30
51
  redis_results = @pool.with do |conn|
31
52
  conn.pipelined do |pipe|
32
- minutes.times do |idx|
33
- key = "j|#{time.strftime("%Y%m%d")}|#{time.hour}:#{time.min}"
53
+ count.times do |idx|
54
+ key = keyproc.call(time)
34
55
  pipe.hgetall key
35
- result.prepend_bucket time
36
- time -= 60
56
+ time -= stride
37
57
  end
38
58
  end
39
59
  end
40
60
 
61
+ result.starts_at = time
41
62
  time = @time
42
63
  redis_results.each do |hash|
43
64
  hash.each do |k, v|
@@ -45,63 +66,66 @@ module Sidekiq
45
66
  next if class_filter && !class_filter.match?(kls)
46
67
  result.job_results[kls].add_metric metric, time, v.to_i
47
68
  end
48
- time -= 60
69
+ time -= stride
49
70
  end
50
71
 
51
- result.marks = fetch_marks(result.starts_at..result.ends_at)
52
-
72
+ result.marks = fetch_marks(result.starts_at..result.ends_at, granularity)
53
73
  result
54
74
  end
55
75
 
56
- def for_job(klass, minutes: 60)
57
- result = Result.new
58
-
76
+ def for_job(klass, minutes: nil, hours: nil)
59
77
  time = @time
78
+ minutes = 60 unless minutes || hours
79
+
80
+ # DoS protection, sanity check
81
+ minutes = 60 if minutes && minutes > 480
82
+ hours = 72 if hours && hours > 72
83
+
84
+ granularity = hours ? :hourly : :minutely
85
+ result = Result.new(granularity)
86
+ result.ends_at = time
87
+ count = hours ? hours * 6 : minutes
88
+ stride, keyproc = ROLLUPS[granularity]
89
+
60
90
  redis_results = @pool.with do |conn|
61
91
  conn.pipelined do |pipe|
62
- minutes.times do |idx|
63
- key = "j|#{time.strftime("%Y%m%d")}|#{time.hour}:#{time.min}"
92
+ count.times do |idx|
93
+ key = keyproc.call(time)
64
94
  pipe.hmget key, "#{klass}|ms", "#{klass}|p", "#{klass}|f"
65
- result.prepend_bucket time
66
- time -= 60
95
+ time -= stride
67
96
  end
68
97
  end
69
98
  end
70
99
 
100
+ result.starts_at = time
71
101
  time = @time
72
102
  @pool.with do |conn|
73
103
  redis_results.each do |(ms, p, f)|
74
104
  result.job_results[klass].add_metric "ms", time, ms.to_i if ms
75
105
  result.job_results[klass].add_metric "p", time, p.to_i if p
76
106
  result.job_results[klass].add_metric "f", time, f.to_i if f
77
- result.job_results[klass].add_hist time, Histogram.new(klass).fetch(conn, time).reverse
78
- time -= 60
107
+ result.job_results[klass].add_hist time, Histogram.new(klass).fetch(conn, time).reverse if minutes
108
+ time -= stride
79
109
  end
80
110
  end
81
111
 
82
- result.marks = fetch_marks(result.starts_at..result.ends_at)
83
-
112
+ result.marks = fetch_marks(result.starts_at..result.ends_at, granularity)
84
113
  result
85
114
  end
86
115
 
87
- class Result < Struct.new(:starts_at, :ends_at, :size, :buckets, :job_results, :marks)
88
- def initialize
116
+ class Result < Struct.new(:granularity, :starts_at, :ends_at, :size, :job_results, :marks)
117
+ def initialize(granularity = :minutely)
89
118
  super
90
- self.buckets = []
119
+ self.granularity = granularity
91
120
  self.marks = []
92
- self.job_results = Hash.new { |h, k| h[k] = JobResult.new }
93
- end
94
-
95
- def prepend_bucket(time)
96
- buckets.unshift time.strftime("%H:%M")
97
- self.ends_at ||= time
98
- self.starts_at = time
121
+ self.job_results = Hash.new { |h, k| h[k] = JobResult.new(granularity) }
99
122
  end
100
123
  end
101
124
 
102
- class JobResult < Struct.new(:series, :hist, :totals)
103
- def initialize
125
+ class JobResult < Struct.new(:granularity, :series, :hist, :totals)
126
+ def initialize(granularity = :minutely)
104
127
  super
128
+ self.granularity = granularity
105
129
  self.series = Hash.new { |h, k| h[k] = Hash.new(0) }
106
130
  self.hist = Hash.new { |h, k| h[k] = [] }
107
131
  self.totals = Hash.new(0)
@@ -109,14 +133,14 @@ module Sidekiq
109
133
 
110
134
  def add_metric(metric, time, value)
111
135
  totals[metric] += value
112
- series[metric][time.strftime("%H:%M")] += value
136
+ series[metric][Query.bkt_time_s(time, granularity)] += value
113
137
 
114
138
  # Include timing measurements in seconds for convenience
115
139
  add_metric("s", time, value / 1000.0) if metric == "ms"
116
140
  end
117
141
 
118
142
  def add_hist(time, hist_result)
119
- hist[time.strftime("%H:%M")] = hist_result
143
+ hist[Query.bkt_time_s(time, granularity)] = hist_result
120
144
  end
121
145
 
122
146
  def total_avg(metric = "ms")
@@ -133,22 +157,24 @@ module Sidekiq
133
157
  end
134
158
  end
135
159
 
136
- class MarkResult < Struct.new(:time, :label)
137
- def bucket
138
- time.strftime("%H:%M")
139
- end
160
+ MarkResult = Struct.new(:time, :label, :bucket)
161
+
162
+ def self.bkt_time_s(time, granularity)
163
+ # truncate time to ten minutes ("8:40", not "8:43") or one minute
164
+ truncation = (granularity == :hourly) ? 600 : 60
165
+ Time.at(time.to_i - time.to_i % truncation).utc.iso8601
140
166
  end
141
167
 
142
168
  private
143
169
 
144
- def fetch_marks(time_range)
170
+ def fetch_marks(time_range, granularity)
145
171
  [].tap do |result|
146
172
  marks = @pool.with { |c| c.hgetall("#{@time.strftime("%Y%m%d")}-marks") }
147
173
 
148
174
  marks.each do |timestamp, label|
149
175
  time = Time.parse(timestamp)
150
176
  if time_range.cover? time
151
- result << MarkResult.new(time, label)
177
+ result << MarkResult.new(time, label, Query.bkt_time_s(time, granularity))
152
178
  end
153
179
  end
154
180
  end
@@ -25,7 +25,10 @@ module Sidekiq
25
25
  #
26
26
  # To store this data, we use Redis' BITFIELD command to store unsigned 16-bit counters
27
27
  # per bucket per klass per minute. It's unlikely that most people will be executing more
28
- # than 1000 job/sec for a full minute of a specific type.
28
+ # than 1000 job/sec for a full minute of a specific type (i.e. overflow 65,536).
29
+ #
30
+ # Histograms are only stored at the fine-grained level, they are not rolled up
31
+ # for longer-term buckets.
29
32
  class Histogram
30
33
  include Enumerable
31
34
 
@@ -82,15 +85,15 @@ module Sidekiq
82
85
  end
83
86
 
84
87
  def fetch(conn, now = Time.now)
85
- window = now.utc.strftime("%d-%H:%-M")
86
- key = "#{@klass}-#{window}"
88
+ window = now.utc.strftime("%-d-%-H:%-M")
89
+ key = "h|#{@klass}-#{window}"
87
90
  conn.bitfield_ro(key, *FETCH)
88
91
  end
89
92
 
90
93
  def persist(conn, now = Time.now)
91
94
  buckets, @buckets = @buckets, []
92
- window = now.utc.strftime("%d-%H:%-M")
93
- key = "#{@klass}-#{window}"
95
+ window = now.utc.strftime("%-d-%-H:%-M")
96
+ key = "h|#{@klass}-#{window}"
94
97
  cmd = [key, "OVERFLOW", "SAT"]
95
98
  buckets.each_with_index do |counter, idx|
96
99
  val = counter.value
@@ -19,13 +19,13 @@ module Sidekiq
19
19
  end
20
20
 
21
21
  def track(queue, klass)
22
- start = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
22
+ start = mono_ms
23
23
  time_ms = 0
24
24
  begin
25
25
  begin
26
26
  yield
27
27
  ensure
28
- finish = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC, :millisecond)
28
+ finish = mono_ms
29
29
  time_ms = finish - start
30
30
  end
31
31
  # We don't track time for failed jobs as they can have very unpredictable
@@ -51,7 +51,7 @@ module Sidekiq
51
51
  end
52
52
 
53
53
  # LONG_TERM = 90 * 24 * 60 * 60
54
- # MID_TERM = 7 * 24 * 60 * 60
54
+ MID_TERM = 3 * 24 * 60 * 60
55
55
  SHORT_TERM = 8 * 60 * 60
56
56
 
57
57
  def flush(time = Time.now)
@@ -62,8 +62,10 @@ module Sidekiq
62
62
 
63
63
  now = time.utc
64
64
  # nowdate = now.strftime("%Y%m%d")
65
- # nowhour = now.strftime("%Y%m%d|%-H")
66
- nowmin = now.strftime("%Y%m%d|%-H:%-M")
65
+ # "250214|8:4" is the 10 minute bucket for Feb 14 2025, 08:43
66
+ nowmid = now.strftime("%y%m%d|%-H:%M")[0..-2]
67
+ # "250214|8:43" is the 1 minute bucket for Feb 14 2025, 08:43
68
+ nowshort = now.strftime("%y%m%d|%-H:%M")
67
69
  count = 0
68
70
 
69
71
  redis do |conn|
@@ -81,8 +83,8 @@ module Sidekiq
81
83
  # daily or hourly rollups.
82
84
  [
83
85
  # ["j", jobs, nowdate, LONG_TERM],
84
- # ["j", jobs, nowhour, MID_TERM],
85
- ["j", jobs, nowmin, SHORT_TERM]
86
+ ["j", jobs, nowmid, MID_TERM],
87
+ ["j", jobs, nowshort, SHORT_TERM]
86
88
  ].each do |prefix, data, bucket, ttl|
87
89
  conn.pipelined do |xa|
88
90
  stats = "#{prefix}|#{bucket}"
@@ -1,5 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "active_job/arguments"
3
4
  require "active_support/current_attributes"
4
5
 
5
6
  module Sidekiq
@@ -20,6 +21,8 @@ module Sidekiq
20
21
  # Sidekiq::CurrentAttributes.persist(["Myapp::Current", "Myapp::OtherCurrent"])
21
22
  #
22
23
  module CurrentAttributes
24
+ Serializer = ::ActiveJob::Arguments
25
+
23
26
  class Save
24
27
  include Sidekiq::ClientMiddleware
25
28
 
@@ -33,26 +36,11 @@ module Sidekiq
33
36
  attrs = strklass.constantize.attributes
34
37
  # Retries can push the job N times, we don't
35
38
  # want retries to reset cattr. #5692, #5090
36
- if attrs.any?
37
- # Older rails has a bug that `CurrentAttributes#attributes` always returns
38
- # the same hash instance. We need to dup it to avoid being accidentally mutated.
39
- job[key] = if returns_same_object?
40
- attrs.dup
41
- else
42
- attrs
43
- end
44
- end
39
+ job[key] = Serializer.serialize(attrs) if attrs.any?
45
40
  end
46
41
  end
47
42
  yield
48
43
  end
49
-
50
- private
51
-
52
- def returns_same_object?
53
- ActiveSupport::VERSION::MAJOR < 8 ||
54
- (ActiveSupport::VERSION::MAJOR == 8 && ActiveSupport::VERSION::MINOR == 0)
55
- end
56
44
  end
57
45
 
58
46
  class Load
@@ -68,7 +56,7 @@ module Sidekiq
68
56
  @cattrs.each do |(key, strklass)|
69
57
  next unless job.has_key?(key)
70
58
 
71
- klass_attrs[strklass.constantize] = job[key]
59
+ klass_attrs[strklass.constantize] = Serializer.deserialize(job[key]).to_h
72
60
  end
73
61
 
74
62
  wrap(klass_attrs.to_a, &block)
@@ -17,7 +17,14 @@ module Sidekiq
17
17
  ending = starting + page_size - 1
18
18
 
19
19
  Sidekiq.redis do |conn|
20
- type = conn.type(key)
20
+ # horrible, think you can make this cleaner?
21
+ type = TYPE_CACHE[key]
22
+ if type
23
+ elsif key.start_with?("queue:")
24
+ type = TYPE_CACHE[key] = "list"
25
+ else
26
+ type = TYPE_CACHE[key] = conn.type(key)
27
+ end
21
28
  rev = opts && opts[:reverse]
22
29
 
23
30
  case type
@@ -3,6 +3,7 @@
3
3
  require "sidekiq/fetch"
4
4
  require "sidekiq/job_logger"
5
5
  require "sidekiq/job_retry"
6
+ require "sidekiq/profiler"
6
7
 
7
8
  module Sidekiq
8
9
  ##
@@ -66,7 +67,7 @@ module Sidekiq
66
67
  @thread ||= safe_thread("#{config.name}/processor", &method(:run))
67
68
  end
68
69
 
69
- private unless $TESTING
70
+ private
70
71
 
71
72
  def run
72
73
  # By setting this thread-local, Sidekiq.redis will access +Sidekiq::Capsule#redis_pool+
@@ -112,13 +113,17 @@ module Sidekiq
112
113
  def handle_fetch_exception(ex)
113
114
  unless @down
114
115
  @down = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
115
- logger.error("Error fetching job: #{ex}")
116
116
  handle_exception(ex)
117
117
  end
118
118
  sleep(1)
119
119
  nil
120
120
  end
121
121
 
122
+ def profile(job, &block)
123
+ return yield unless job["profile"]
124
+ Sidekiq::Profiler.new(config).call(job, &block)
125
+ end
126
+
122
127
  def dispatch(job_hash, queue, jobstr)
123
128
  # since middleware can mutate the job hash
124
129
  # we need to clone it to report the original
@@ -132,17 +137,19 @@ module Sidekiq
132
137
  @retrier.global(jobstr, queue) do
133
138
  @job_logger.call(job_hash, queue) do
134
139
  stats(jobstr, queue) do
135
- # Rails 5 requires a Reloader to wrap code execution. In order to
136
- # constantize the worker and instantiate an instance, we have to call
137
- # the Reloader. It handles code loading, db connection management, etc.
138
- # Effectively this block denotes a "unit of work" to Rails.
139
- @reloader.call do
140
- klass = Object.const_get(job_hash["class"])
141
- instance = klass.new
142
- instance.jid = job_hash["jid"]
143
- instance._context = self
144
- @retrier.local(instance, jobstr, queue) do
145
- yield instance
140
+ profile(job_hash) do
141
+ # Rails 5 requires a Reloader to wrap code execution. In order to
142
+ # constantize the worker and instantiate an instance, we have to call
143
+ # the Reloader. It handles code loading, db connection management, etc.
144
+ # Effectively this block denotes a "unit of work" to Rails.
145
+ @reloader.call do
146
+ klass = Object.const_get(job_hash["class"])
147
+ instance = klass.new
148
+ instance.jid = job_hash["jid"]
149
+ instance._context = self
150
+ @retrier.local(instance, jobstr, queue) do
151
+ yield instance
152
+ end
146
153
  end
147
154
  end
148
155
  end
@@ -165,7 +172,6 @@ module Sidekiq
165
172
  begin
166
173
  job_hash = Sidekiq.load_json(jobstr)
167
174
  rescue => ex
168
- handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
169
175
  now = Time.now.to_f
170
176
  redis do |conn|
171
177
  conn.multi do |xa|
@@ -174,6 +180,7 @@ module Sidekiq
174
180
  xa.zremrangebyrank("dead", 0, - @capsule.config[:dead_max_jobs])
175
181
  end
176
182
  end
183
+ handle_exception(ex, {context: "Invalid JSON for job", jobstr: jobstr})
177
184
  return uow.acknowledge
178
185
  end
179
186