sidekiq 5.2.10 → 6.5.6
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +391 -1
- data/LICENSE +3 -3
- data/README.md +24 -35
- data/bin/sidekiq +27 -3
- data/bin/sidekiqload +79 -67
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +504 -307
- data/lib/sidekiq/cli.rb +190 -206
- data/lib/sidekiq/client.rb +77 -81
- data/lib/sidekiq/component.rb +65 -0
- data/lib/sidekiq/delay.rb +8 -7
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +7 -5
- data/lib/sidekiq/fetch.rb +50 -40
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +33 -7
- data/lib/sidekiq/job_retry.rb +126 -106
- data/lib/sidekiq/job_util.rb +71 -0
- data/lib/sidekiq/launcher.rb +177 -83
- data/lib/sidekiq/logger.rb +156 -0
- data/lib/sidekiq/manager.rb +40 -41
- data/lib/sidekiq/metrics/deploy.rb +47 -0
- data/lib/sidekiq/metrics/query.rb +153 -0
- data/lib/sidekiq/metrics/shared.rb +94 -0
- data/lib/sidekiq/metrics/tracking.rb +134 -0
- data/lib/sidekiq/middleware/chain.rb +102 -46
- data/lib/sidekiq/middleware/current_attributes.rb +63 -0
- data/lib/sidekiq/middleware/i18n.rb +7 -7
- data/lib/sidekiq/middleware/modules.rb +21 -0
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +20 -16
- data/lib/sidekiq/processor.rb +104 -97
- data/lib/sidekiq/rails.rb +47 -37
- data/lib/sidekiq/redis_client_adapter.rb +154 -0
- data/lib/sidekiq/redis_connection.rb +108 -77
- data/lib/sidekiq/ring_buffer.rb +29 -0
- data/lib/sidekiq/scheduled.rb +64 -35
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +6 -5
- data/lib/sidekiq/testing.rb +68 -58
- data/lib/sidekiq/transaction_aware_client.rb +45 -0
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +100 -77
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +134 -94
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +65 -105
- data/lib/sidekiq/worker.rb +253 -106
- data/lib/sidekiq.rb +170 -62
- data/sidekiq.gemspec +23 -16
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +112 -61
- data/web/assets/javascripts/chart.min.js +13 -0
- data/web/assets/javascripts/chartjs-plugin-annotation.min.js +7 -0
- data/web/assets/javascripts/dashboard.js +53 -89
- data/web/assets/javascripts/graph.js +16 -0
- data/web/assets/javascripts/metrics.js +262 -0
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +88 -233
- data/web/locales/ar.yml +8 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/el.yml +43 -19
- data/web/locales/en.yml +13 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +7 -1
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/pt-br.yml +27 -9
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +1 -1
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +56 -22
- data/web/views/dashboard.erb +23 -14
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +3 -1
- data/web/views/metrics.erb +69 -0
- data/web/views/metrics_for_job.erb +87 -0
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +23 -10
- data/web/views/queues.erb +10 -2
- data/web/views/retries.erb +11 -8
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +5 -2
- metadata +53 -64
- data/.circleci/config.yml +0 -61
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -15
- data/.travis.yml +0 -11
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/COMM-LICENSE +0 -97
- data/Ent-Changes.md +0 -238
- data/Gemfile +0 -19
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-Changes.md +0 -759
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -20
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/ctl.rb +0 -221
- data/lib/sidekiq/exception_handler.rb +0 -29
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
- data/lib/sidekiq/util.rb +0 -66
data/bin/sidekiqload
CHANGED
@@ -4,18 +4,22 @@
|
|
4
4
|
# RUBYOPT=-w bundle exec sidekiq
|
5
5
|
$TESTING = false
|
6
6
|
|
7
|
-
#require
|
8
|
-
|
7
|
+
# require "ruby-prof"
|
8
|
+
require "bundler/setup"
|
9
|
+
Bundler.require(:default, :load_test)
|
9
10
|
|
10
|
-
require_relative
|
11
|
-
require_relative
|
11
|
+
require_relative "../lib/sidekiq/cli"
|
12
|
+
require_relative "../lib/sidekiq/launcher"
|
12
13
|
|
13
|
-
|
14
|
+
if ENV["SIDEKIQ_REDIS_CLIENT"]
|
15
|
+
Sidekiq::RedisConnection.adapter = :redis_client
|
16
|
+
end
|
14
17
|
|
15
18
|
Sidekiq.configure_server do |config|
|
16
|
-
|
17
|
-
config.redis = {
|
18
|
-
config.
|
19
|
+
config.options[:concurrency] = 10
|
20
|
+
config.redis = {db: 13, port: 6380}
|
21
|
+
# config.redis = { db: 13, port: 6380, driver: :hiredis}
|
22
|
+
config.options[:queues] << "default"
|
19
23
|
config.logger.level = Logger::ERROR
|
20
24
|
config.average_scheduled_poll_interval = 2
|
21
25
|
config.reliable! if defined?(Sidekiq::Pro)
|
@@ -28,51 +32,50 @@ class LoadWorker
|
|
28
32
|
1
|
29
33
|
end
|
30
34
|
|
31
|
-
def perform(idx)
|
32
|
-
|
35
|
+
def perform(idx, ts = nil)
|
36
|
+
puts(Time.now.to_f - ts) if !ts.nil?
|
37
|
+
# raise idx.to_s if idx % 100 == 1
|
33
38
|
end
|
34
39
|
end
|
35
40
|
|
36
41
|
# brew tap shopify/shopify
|
37
42
|
# brew install toxiproxy
|
38
|
-
#
|
39
|
-
|
43
|
+
# run `toxiproxy-server` in a separate terminal window.
|
44
|
+
require "toxiproxy"
|
40
45
|
# simulate a non-localhost network for realer-world conditions.
|
41
46
|
# adding 1ms of network latency has an ENORMOUS impact on benchmarks
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
+
Toxiproxy.populate([{
|
48
|
+
name: "redis",
|
49
|
+
listen: "127.0.0.1:6380",
|
50
|
+
upstream: "127.0.0.1:6379"
|
51
|
+
}])
|
47
52
|
|
48
53
|
self_read, self_write = IO.pipe
|
49
|
-
%w
|
50
|
-
|
51
|
-
|
52
|
-
self_write.puts(sig)
|
53
|
-
end
|
54
|
-
rescue ArgumentError
|
55
|
-
puts "Signal #{sig} not supported"
|
54
|
+
%w[INT TERM TSTP TTIN].each do |sig|
|
55
|
+
trap sig do
|
56
|
+
self_write.puts(sig)
|
56
57
|
end
|
58
|
+
rescue ArgumentError
|
59
|
+
puts "Signal #{sig} not supported"
|
57
60
|
end
|
58
61
|
|
59
|
-
Sidekiq.redis {|c| c.flushdb}
|
62
|
+
Sidekiq.redis { |c| c.flushdb }
|
60
63
|
def handle_signal(launcher, sig)
|
61
64
|
Sidekiq.logger.debug "Got #{sig} signal"
|
62
65
|
case sig
|
63
|
-
when
|
66
|
+
when "INT"
|
64
67
|
# Handle Ctrl-C in JRuby like MRI
|
65
68
|
# http://jira.codehaus.org/browse/JRUBY-4637
|
66
69
|
raise Interrupt
|
67
|
-
when
|
70
|
+
when "TERM"
|
68
71
|
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
69
72
|
raise Interrupt
|
70
|
-
when
|
73
|
+
when "TSTP"
|
71
74
|
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
72
75
|
launcher.quiet
|
73
|
-
when
|
76
|
+
when "TTIN"
|
74
77
|
Thread.list.each do |thread|
|
75
|
-
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread[
|
78
|
+
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["label"]}"
|
76
79
|
if thread.backtrace
|
77
80
|
Sidekiq.logger.warn thread.backtrace.join("\n")
|
78
81
|
else
|
@@ -86,47 +89,56 @@ def Process.rss
|
|
86
89
|
`ps -o rss= -p #{Process.pid}`.chomp.to_i
|
87
90
|
end
|
88
91
|
|
89
|
-
iter =
|
92
|
+
iter = 50
|
90
93
|
count = 10_000
|
91
94
|
|
92
95
|
iter.times do
|
93
|
-
arr = Array.new(count)
|
94
|
-
|
95
|
-
end
|
96
|
-
count.times do |idx|
|
97
|
-
arr[idx][0] = idx
|
98
|
-
end
|
99
|
-
Sidekiq::Client.push_bulk('class' => LoadWorker, 'args' => arr)
|
96
|
+
arr = Array.new(count) { |idx| [idx] }
|
97
|
+
Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
|
100
98
|
end
|
101
|
-
Sidekiq.logger.error "Created #{count*iter} jobs"
|
99
|
+
Sidekiq.logger.error "Created #{count * iter} jobs"
|
100
|
+
|
101
|
+
start = Time.now
|
102
102
|
|
103
103
|
Monitoring = Thread.new do
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
104
|
+
while true
|
105
|
+
sleep 0.2
|
106
|
+
qsize = Sidekiq.redis do |conn|
|
107
|
+
conn.llen "queue:default"
|
108
|
+
end
|
109
|
+
total = qsize
|
110
|
+
# Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
|
111
|
+
if total == 0
|
112
|
+
Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
|
113
|
+
Sidekiq.logger.error("Now here's the latency for three jobs")
|
114
|
+
|
115
|
+
LoadWorker.perform_async(1, Time.now.to_f)
|
116
|
+
LoadWorker.perform_async(2, Time.now.to_f)
|
117
|
+
LoadWorker.perform_async(3, Time.now.to_f)
|
118
|
+
|
119
|
+
sleep 0.2
|
120
|
+
exit(0)
|
120
121
|
end
|
121
122
|
end
|
122
123
|
end
|
123
124
|
|
125
|
+
def with_latency(latency, &block)
|
126
|
+
Sidekiq.logger.error "Simulating #{latency}ms of latency between Sidekiq and redis"
|
127
|
+
if latency > 0
|
128
|
+
Toxiproxy[:redis].downstream(:latency, latency: latency).apply(&block)
|
129
|
+
else
|
130
|
+
yield
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
124
134
|
begin
|
125
|
-
#RubyProf::exclude_threads = [ Monitoring ]
|
126
|
-
#RubyProf.start
|
127
|
-
|
128
|
-
|
129
|
-
|
135
|
+
# RubyProf::exclude_threads = [ Monitoring ]
|
136
|
+
# RubyProf.start
|
137
|
+
events = Sidekiq.options[:lifecycle_events][:startup]
|
138
|
+
events.each(&:call)
|
139
|
+
events.clear
|
140
|
+
|
141
|
+
with_latency(Integer(ENV.fetch("LATENCY", "1"))) do
|
130
142
|
launcher = Sidekiq::Launcher.new(Sidekiq.options)
|
131
143
|
launcher.run
|
132
144
|
|
@@ -134,16 +146,16 @@ begin
|
|
134
146
|
signal = readable_io.first[0].gets.strip
|
135
147
|
handle_signal(launcher, signal)
|
136
148
|
end
|
137
|
-
|
149
|
+
end
|
138
150
|
rescue SystemExit => e
|
139
|
-
#Sidekiq.logger.error("Profiling...")
|
140
|
-
#result = RubyProf.stop
|
141
|
-
#printer = RubyProf::GraphHtmlPrinter.new(result)
|
142
|
-
#printer.print(File.new("output.html", "w"), :min_percent => 1)
|
151
|
+
# Sidekiq.logger.error("Profiling...")
|
152
|
+
# result = RubyProf.stop
|
153
|
+
# printer = RubyProf::GraphHtmlPrinter.new(result)
|
154
|
+
# printer.print(File.new("output.html", "w"), :min_percent => 1)
|
143
155
|
# normal
|
144
156
|
rescue => e
|
145
157
|
raise e if $DEBUG
|
146
|
-
|
147
|
-
|
158
|
+
warn e.message
|
159
|
+
warn e.backtrace.join("\n")
|
148
160
|
exit 1
|
149
161
|
end
|
data/bin/sidekiqmon
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
require "rails/generators/named_base"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
module Generators # :nodoc:
|
5
|
+
class JobGenerator < ::Rails::Generators::NamedBase # :nodoc:
|
6
|
+
desc "This generator creates a Sidekiq Job in app/sidekiq and a corresponding test"
|
7
|
+
|
8
|
+
check_class_collision suffix: "Job"
|
9
|
+
|
10
|
+
def self.default_generator_root
|
11
|
+
File.dirname(__FILE__)
|
12
|
+
end
|
13
|
+
|
14
|
+
def create_job_file
|
15
|
+
template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
|
16
|
+
end
|
17
|
+
|
18
|
+
def create_test_file
|
19
|
+
return unless test_framework
|
20
|
+
|
21
|
+
if test_framework == :rspec
|
22
|
+
create_job_spec
|
23
|
+
else
|
24
|
+
create_job_test
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def create_job_spec
|
31
|
+
template_file = File.join(
|
32
|
+
"spec/sidekiq",
|
33
|
+
class_path,
|
34
|
+
"#{file_name}_job_spec.rb"
|
35
|
+
)
|
36
|
+
template "job_spec.rb.erb", template_file
|
37
|
+
end
|
38
|
+
|
39
|
+
def create_job_test
|
40
|
+
template_file = File.join(
|
41
|
+
"test/sidekiq",
|
42
|
+
class_path,
|
43
|
+
"#{file_name}_job_test.rb"
|
44
|
+
)
|
45
|
+
template "job_test.rb.erb", template_file
|
46
|
+
end
|
47
|
+
|
48
|
+
def file_name
|
49
|
+
@_file_name ||= super.sub(/_?job\z/i, "")
|
50
|
+
end
|
51
|
+
|
52
|
+
def test_framework
|
53
|
+
::Rails.application.config.generators.options[:rails][:test_framework]
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'test_helper'
|
2
2
|
<% module_namespacing do -%>
|
3
|
-
class <%= class_name %>
|
3
|
+
class <%= class_name %>JobTest < Minitest::Test
|
4
4
|
def test_example
|
5
5
|
skip "add some examples to (or delete) #{__FILE__}"
|
6
6
|
end
|