sidekiq 5.2.9 → 6.4.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/Changes.md +318 -1
- data/LICENSE +3 -3
- data/README.md +23 -34
- data/bin/sidekiq +27 -3
- data/bin/sidekiqload +67 -61
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/job_generator.rb +57 -0
- data/lib/generators/sidekiq/templates/{worker.rb.erb → job.rb.erb} +2 -2
- data/lib/generators/sidekiq/templates/{worker_spec.rb.erb → job_spec.rb.erb} +1 -1
- data/lib/generators/sidekiq/templates/{worker_test.rb.erb → job_test.rb.erb} +1 -1
- data/lib/sidekiq/api.rb +335 -267
- data/lib/sidekiq/cli.rb +164 -182
- data/lib/sidekiq/client.rb +58 -61
- data/lib/sidekiq/delay.rb +7 -6
- data/lib/sidekiq/exception_handler.rb +10 -12
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +6 -4
- data/lib/sidekiq/fetch.rb +40 -32
- data/lib/sidekiq/job.rb +13 -0
- data/lib/sidekiq/job_logger.rb +33 -7
- data/lib/sidekiq/job_retry.rb +70 -71
- data/lib/sidekiq/job_util.rb +65 -0
- data/lib/sidekiq/launcher.rb +161 -71
- data/lib/sidekiq/logger.rb +170 -0
- data/lib/sidekiq/manager.rb +17 -21
- data/lib/sidekiq/middleware/chain.rb +20 -8
- data/lib/sidekiq/middleware/current_attributes.rb +57 -0
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +20 -16
- data/lib/sidekiq/processor.rb +71 -70
- data/lib/sidekiq/rails.rb +40 -37
- data/lib/sidekiq/redis_connection.rb +48 -48
- data/lib/sidekiq/scheduled.rb +62 -28
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/testing.rb +36 -27
- data/lib/sidekiq/util.rb +57 -15
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -11
- data/lib/sidekiq/web/application.rb +88 -75
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +109 -92
- data/lib/sidekiq/web/router.rb +23 -19
- data/lib/sidekiq/web.rb +61 -105
- data/lib/sidekiq/worker.rb +247 -105
- data/lib/sidekiq.rb +77 -44
- data/sidekiq.gemspec +23 -16
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +83 -64
- data/web/assets/javascripts/dashboard.js +54 -73
- data/web/assets/stylesheets/application-dark.css +143 -0
- data/web/assets/stylesheets/application-rtl.css +0 -4
- data/web/assets/stylesheets/application.css +45 -232
- data/web/locales/ar.yml +8 -2
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +6 -1
- data/web/locales/es.yml +18 -2
- data/web/locales/fr.yml +10 -3
- data/web/locales/ja.yml +7 -1
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +1 -1
- data/web/views/_job_info.erb +3 -2
- data/web/views/_poll_link.erb +2 -5
- data/web/views/_summary.erb +7 -7
- data/web/views/busy.erb +54 -20
- data/web/views/dashboard.erb +22 -14
- data/web/views/dead.erb +3 -3
- data/web/views/layout.erb +3 -1
- data/web/views/morgue.erb +9 -6
- data/web/views/queue.erb +19 -10
- data/web/views/queues.erb +10 -2
- data/web/views/retries.erb +11 -8
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +5 -2
- metadata +34 -64
- data/.circleci/config.yml +0 -61
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -11
- data/.gitignore +0 -15
- data/.travis.yml +0 -11
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/5.0-Upgrade.md +0 -56
- data/COMM-LICENSE +0 -97
- data/Ent-Changes.md +0 -238
- data/Gemfile +0 -23
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-4.0-Upgrade.md +0 -35
- data/Pro-Changes.md +0 -759
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -20
- data/code_of_conduct.md +0 -50
- data/lib/generators/sidekiq/worker_generator.rb +0 -49
- data/lib/sidekiq/core_ext.rb +0 -1
- data/lib/sidekiq/ctl.rb +0 -221
- data/lib/sidekiq/logging.rb +0 -122
- data/lib/sidekiq/middleware/server/active_record.rb +0 -23
data/bin/sidekiqload
CHANGED
@@ -4,18 +4,18 @@
|
|
4
4
|
# RUBYOPT=-w bundle exec sidekiq
|
5
5
|
$TESTING = false
|
6
6
|
|
7
|
-
#require
|
8
|
-
|
7
|
+
# require "ruby-prof"
|
8
|
+
require "bundler/setup"
|
9
|
+
Bundler.require(:default, :load_test)
|
9
10
|
|
10
|
-
require_relative
|
11
|
-
require_relative
|
12
|
-
|
13
|
-
include Sidekiq::Util
|
11
|
+
require_relative "../lib/sidekiq/cli"
|
12
|
+
require_relative "../lib/sidekiq/launcher"
|
14
13
|
|
15
14
|
Sidekiq.configure_server do |config|
|
16
|
-
|
17
|
-
config.redis = {
|
18
|
-
config.
|
15
|
+
config.options[:concurrency] = 10
|
16
|
+
config.redis = {db: 13, port: 6380}
|
17
|
+
# config.redis = { db: 13, port: 6380, driver: :hiredis}
|
18
|
+
config.options[:queues] << "default"
|
19
19
|
config.logger.level = Logger::ERROR
|
20
20
|
config.average_scheduled_poll_interval = 2
|
21
21
|
config.reliable! if defined?(Sidekiq::Pro)
|
@@ -28,51 +28,51 @@ class LoadWorker
|
|
28
28
|
1
|
29
29
|
end
|
30
30
|
|
31
|
-
def perform(idx)
|
32
|
-
|
31
|
+
def perform(idx, ts = nil)
|
32
|
+
puts(Time.now.to_f - ts) if !ts.nil?
|
33
|
+
# raise idx.to_s if idx % 100 == 1
|
33
34
|
end
|
34
35
|
end
|
35
36
|
|
36
37
|
# brew tap shopify/shopify
|
37
38
|
# brew install toxiproxy
|
38
39
|
# gem install toxiproxy
|
39
|
-
#
|
40
|
+
# run `toxiproxy-server` in a separate terminal window.
|
41
|
+
require "toxiproxy"
|
40
42
|
# simulate a non-localhost network for realer-world conditions.
|
41
43
|
# adding 1ms of network latency has an ENORMOUS impact on benchmarks
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
44
|
+
Toxiproxy.populate([{
|
45
|
+
name: "redis",
|
46
|
+
listen: "127.0.0.1:6380",
|
47
|
+
upstream: "127.0.0.1:6379"
|
48
|
+
}])
|
47
49
|
|
48
50
|
self_read, self_write = IO.pipe
|
49
|
-
%w
|
50
|
-
|
51
|
-
|
52
|
-
self_write.puts(sig)
|
53
|
-
end
|
54
|
-
rescue ArgumentError
|
55
|
-
puts "Signal #{sig} not supported"
|
51
|
+
%w[INT TERM TSTP TTIN].each do |sig|
|
52
|
+
trap sig do
|
53
|
+
self_write.puts(sig)
|
56
54
|
end
|
55
|
+
rescue ArgumentError
|
56
|
+
puts "Signal #{sig} not supported"
|
57
57
|
end
|
58
58
|
|
59
|
-
Sidekiq.redis {|c| c.flushdb}
|
59
|
+
Sidekiq.redis { |c| c.flushdb }
|
60
60
|
def handle_signal(launcher, sig)
|
61
61
|
Sidekiq.logger.debug "Got #{sig} signal"
|
62
62
|
case sig
|
63
|
-
when
|
63
|
+
when "INT"
|
64
64
|
# Handle Ctrl-C in JRuby like MRI
|
65
65
|
# http://jira.codehaus.org/browse/JRUBY-4637
|
66
66
|
raise Interrupt
|
67
|
-
when
|
67
|
+
when "TERM"
|
68
68
|
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
69
69
|
raise Interrupt
|
70
|
-
when
|
70
|
+
when "TSTP"
|
71
71
|
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
72
72
|
launcher.quiet
|
73
|
-
when
|
73
|
+
when "TTIN"
|
74
74
|
Thread.list.each do |thread|
|
75
|
-
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread[
|
75
|
+
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread["label"]}"
|
76
76
|
if thread.backtrace
|
77
77
|
Sidekiq.logger.warn thread.backtrace.join("\n")
|
78
78
|
else
|
@@ -86,7 +86,7 @@ def Process.rss
|
|
86
86
|
`ps -o rss= -p #{Process.pid}`.chomp.to_i
|
87
87
|
end
|
88
88
|
|
89
|
-
iter =
|
89
|
+
iter = 50
|
90
90
|
count = 10_000
|
91
91
|
|
92
92
|
iter.times do
|
@@ -96,37 +96,43 @@ iter.times do
|
|
96
96
|
count.times do |idx|
|
97
97
|
arr[idx][0] = idx
|
98
98
|
end
|
99
|
-
Sidekiq::Client.push_bulk(
|
99
|
+
Sidekiq::Client.push_bulk("class" => LoadWorker, "args" => arr)
|
100
100
|
end
|
101
|
-
Sidekiq.logger.error "Created #{count*iter} jobs"
|
101
|
+
Sidekiq.logger.error "Created #{count * iter} jobs"
|
102
|
+
|
103
|
+
start = Time.now
|
102
104
|
|
103
105
|
Monitoring = Thread.new do
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
106
|
+
while true
|
107
|
+
sleep 0.2
|
108
|
+
qsize = Sidekiq.redis do |conn|
|
109
|
+
conn.llen "queue:default"
|
110
|
+
end
|
111
|
+
total = qsize
|
112
|
+
# Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
|
113
|
+
if total == 0
|
114
|
+
Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
|
115
|
+
Sidekiq.logger.error("Now here's the latency for three jobs")
|
116
|
+
|
117
|
+
LoadWorker.perform_async(1, Time.now.to_f)
|
118
|
+
LoadWorker.perform_async(2, Time.now.to_f)
|
119
|
+
LoadWorker.perform_async(3, Time.now.to_f)
|
120
|
+
|
121
|
+
sleep 0.2
|
122
|
+
exit(0)
|
120
123
|
end
|
121
124
|
end
|
122
125
|
end
|
123
126
|
|
124
127
|
begin
|
125
|
-
#RubyProf::exclude_threads = [ Monitoring ]
|
126
|
-
#RubyProf.start
|
127
|
-
|
128
|
-
|
129
|
-
|
128
|
+
# RubyProf::exclude_threads = [ Monitoring ]
|
129
|
+
# RubyProf.start
|
130
|
+
events = Sidekiq.options[:lifecycle_events][:startup]
|
131
|
+
events.each(&:call)
|
132
|
+
events.clear
|
133
|
+
|
134
|
+
Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
|
135
|
+
Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
|
130
136
|
launcher = Sidekiq::Launcher.new(Sidekiq.options)
|
131
137
|
launcher.run
|
132
138
|
|
@@ -134,16 +140,16 @@ begin
|
|
134
140
|
signal = readable_io.first[0].gets.strip
|
135
141
|
handle_signal(launcher, signal)
|
136
142
|
end
|
137
|
-
|
143
|
+
end
|
138
144
|
rescue SystemExit => e
|
139
|
-
#Sidekiq.logger.error("Profiling...")
|
140
|
-
#result = RubyProf.stop
|
141
|
-
#printer = RubyProf::GraphHtmlPrinter.new(result)
|
142
|
-
#printer.print(File.new("output.html", "w"), :min_percent => 1)
|
145
|
+
# Sidekiq.logger.error("Profiling...")
|
146
|
+
# result = RubyProf.stop
|
147
|
+
# printer = RubyProf::GraphHtmlPrinter.new(result)
|
148
|
+
# printer.print(File.new("output.html", "w"), :min_percent => 1)
|
143
149
|
# normal
|
144
150
|
rescue => e
|
145
151
|
raise e if $DEBUG
|
146
|
-
|
147
|
-
|
152
|
+
warn e.message
|
153
|
+
warn e.backtrace.join("\n")
|
148
154
|
exit 1
|
149
155
|
end
|
data/bin/sidekiqmon
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
require "rails/generators/named_base"
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
module Generators # :nodoc:
|
5
|
+
class JobGenerator < ::Rails::Generators::NamedBase # :nodoc:
|
6
|
+
desc "This generator creates a Sidekiq Job in app/sidekiq and a corresponding test"
|
7
|
+
|
8
|
+
check_class_collision suffix: "Job"
|
9
|
+
|
10
|
+
def self.default_generator_root
|
11
|
+
File.dirname(__FILE__)
|
12
|
+
end
|
13
|
+
|
14
|
+
def create_job_file
|
15
|
+
template "job.rb.erb", File.join("app/sidekiq", class_path, "#{file_name}_job.rb")
|
16
|
+
end
|
17
|
+
|
18
|
+
def create_test_file
|
19
|
+
return unless test_framework
|
20
|
+
|
21
|
+
if test_framework == :rspec
|
22
|
+
create_job_spec
|
23
|
+
else
|
24
|
+
create_job_test
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def create_job_spec
|
31
|
+
template_file = File.join(
|
32
|
+
"spec/sidekiq",
|
33
|
+
class_path,
|
34
|
+
"#{file_name}_job_spec.rb"
|
35
|
+
)
|
36
|
+
template "job_spec.rb.erb", template_file
|
37
|
+
end
|
38
|
+
|
39
|
+
def create_job_test
|
40
|
+
template_file = File.join(
|
41
|
+
"test/sidekiq",
|
42
|
+
class_path,
|
43
|
+
"#{file_name}_job_test.rb"
|
44
|
+
)
|
45
|
+
template "job_test.rb.erb", template_file
|
46
|
+
end
|
47
|
+
|
48
|
+
def file_name
|
49
|
+
@_file_name ||= super.sub(/_?job\z/i, "")
|
50
|
+
end
|
51
|
+
|
52
|
+
def test_framework
|
53
|
+
::Rails.application.config.generators.options[:rails][:test_framework]
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'test_helper'
|
2
2
|
<% module_namespacing do -%>
|
3
|
-
class <%= class_name %>
|
3
|
+
class <%= class_name %>JobTest < Minitest::Test
|
4
4
|
def test_example
|
5
5
|
skip "add some examples to (or delete) #{__FILE__}"
|
6
6
|
end
|