sidekiq 4.2.4 → 6.2.1
Sign up to get free protection for your applications and to get access to all the features.
Potentially problematic release.
This version of sidekiq might be problematic. Click here for more details.
- checksums.yaml +5 -5
- data/Changes.md +445 -0
- data/LICENSE +1 -1
- data/README.md +21 -34
- data/bin/sidekiq +26 -2
- data/bin/sidekiqload +28 -38
- data/bin/sidekiqmon +8 -0
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +1 -1
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +2 -2
- data/lib/generators/sidekiq/worker_generator.rb +21 -13
- data/lib/sidekiq/api.rb +347 -213
- data/lib/sidekiq/cli.rb +221 -212
- data/lib/sidekiq/client.rb +75 -52
- data/lib/sidekiq/delay.rb +41 -0
- data/lib/sidekiq/exception_handler.rb +12 -16
- data/lib/sidekiq/extensions/action_mailer.rb +13 -22
- data/lib/sidekiq/extensions/active_record.rb +13 -10
- data/lib/sidekiq/extensions/class_methods.rb +14 -11
- data/lib/sidekiq/extensions/generic_proxy.rb +10 -4
- data/lib/sidekiq/fetch.rb +38 -31
- data/lib/sidekiq/job_logger.rb +63 -0
- data/lib/sidekiq/job_retry.rb +263 -0
- data/lib/sidekiq/launcher.rb +169 -70
- data/lib/sidekiq/logger.rb +166 -0
- data/lib/sidekiq/manager.rb +17 -20
- data/lib/sidekiq/middleware/chain.rb +15 -5
- data/lib/sidekiq/middleware/i18n.rb +5 -7
- data/lib/sidekiq/monitor.rb +133 -0
- data/lib/sidekiq/paginator.rb +18 -14
- data/lib/sidekiq/processor.rb +161 -70
- data/lib/sidekiq/rails.rb +30 -73
- data/lib/sidekiq/redis_connection.rb +67 -20
- data/lib/sidekiq/scheduled.rb +61 -35
- data/lib/sidekiq/sd_notify.rb +149 -0
- data/lib/sidekiq/systemd.rb +24 -0
- data/lib/sidekiq/testing/inline.rb +2 -1
- data/lib/sidekiq/testing.rb +54 -26
- data/lib/sidekiq/util.rb +48 -15
- data/lib/sidekiq/version.rb +2 -1
- data/lib/sidekiq/web/action.rb +15 -15
- data/lib/sidekiq/web/application.rb +112 -89
- data/lib/sidekiq/web/csrf_protection.rb +180 -0
- data/lib/sidekiq/web/helpers.rb +153 -73
- data/lib/sidekiq/web/router.rb +27 -19
- data/lib/sidekiq/web.rb +64 -109
- data/lib/sidekiq/worker.rb +164 -41
- data/lib/sidekiq.rb +86 -60
- data/sidekiq.gemspec +24 -22
- data/web/assets/images/apple-touch-icon.png +0 -0
- data/web/assets/javascripts/application.js +25 -27
- data/web/assets/javascripts/dashboard.js +34 -38
- data/web/assets/stylesheets/application-dark.css +160 -0
- data/web/assets/stylesheets/application-rtl.css +246 -0
- data/web/assets/stylesheets/application.css +402 -12
- data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
- data/web/assets/stylesheets/bootstrap.css +2 -2
- data/web/locales/ar.yml +81 -0
- data/web/locales/de.yml +14 -2
- data/web/locales/en.yml +4 -0
- data/web/locales/es.yml +4 -3
- data/web/locales/fa.yml +80 -0
- data/web/locales/fr.yml +3 -3
- data/web/locales/he.yml +79 -0
- data/web/locales/ja.yml +9 -4
- data/web/locales/lt.yml +83 -0
- data/web/locales/pl.yml +4 -4
- data/web/locales/ru.yml +4 -0
- data/web/locales/ur.yml +80 -0
- data/web/locales/vi.yml +83 -0
- data/web/views/_footer.erb +5 -2
- data/web/views/_job_info.erb +3 -2
- data/web/views/_nav.erb +4 -18
- data/web/views/_paging.erb +1 -1
- data/web/views/busy.erb +57 -19
- data/web/views/dashboard.erb +3 -3
- data/web/views/dead.erb +2 -2
- data/web/views/layout.erb +13 -2
- data/web/views/morgue.erb +19 -12
- data/web/views/queue.erb +22 -12
- data/web/views/queues.erb +13 -3
- data/web/views/retries.erb +22 -13
- data/web/views/retry.erb +3 -3
- data/web/views/scheduled.erb +7 -4
- metadata +42 -194
- data/.github/contributing.md +0 -32
- data/.github/issue_template.md +0 -4
- data/.gitignore +0 -12
- data/.travis.yml +0 -12
- data/3.0-Upgrade.md +0 -70
- data/4.0-Upgrade.md +0 -53
- data/COMM-LICENSE +0 -95
- data/Ent-Changes.md +0 -146
- data/Gemfile +0 -29
- data/Pro-2.0-Upgrade.md +0 -138
- data/Pro-3.0-Upgrade.md +0 -44
- data/Pro-Changes.md +0 -585
- data/Rakefile +0 -9
- data/bin/sidekiqctl +0 -99
- data/code_of_conduct.md +0 -50
- data/lib/sidekiq/core_ext.rb +0 -106
- data/lib/sidekiq/logging.rb +0 -106
- data/lib/sidekiq/middleware/server/active_record.rb +0 -13
- data/lib/sidekiq/middleware/server/logging.rb +0 -40
- data/lib/sidekiq/middleware/server/retry_jobs.rb +0 -205
- data/test/config.yml +0 -9
- data/test/env_based_config.yml +0 -11
- data/test/fake_env.rb +0 -1
- data/test/fixtures/en.yml +0 -2
- data/test/helper.rb +0 -75
- data/test/test_actors.rb +0 -138
- data/test/test_api.rb +0 -528
- data/test/test_cli.rb +0 -418
- data/test/test_client.rb +0 -266
- data/test/test_exception_handler.rb +0 -56
- data/test/test_extensions.rb +0 -127
- data/test/test_fetch.rb +0 -50
- data/test/test_launcher.rb +0 -95
- data/test/test_logging.rb +0 -35
- data/test/test_manager.rb +0 -50
- data/test/test_middleware.rb +0 -158
- data/test/test_processor.rb +0 -235
- data/test/test_rails.rb +0 -22
- data/test/test_redis_connection.rb +0 -132
- data/test/test_retry.rb +0 -326
- data/test/test_retry_exhausted.rb +0 -149
- data/test/test_scheduled.rb +0 -115
- data/test/test_scheduling.rb +0 -58
- data/test/test_sidekiq.rb +0 -107
- data/test/test_testing.rb +0 -143
- data/test/test_testing_fake.rb +0 -357
- data/test/test_testing_inline.rb +0 -94
- data/test/test_util.rb +0 -13
- data/test/test_web.rb +0 -726
- data/test/test_web_helpers.rb +0 -54
data/bin/sidekiqload
CHANGED
@@ -5,29 +5,17 @@
|
|
5
5
|
$TESTING = false
|
6
6
|
|
7
7
|
#require 'ruby-prof'
|
8
|
-
|
8
|
+
require 'bundler/setup'
|
9
|
+
Bundler.require(:default, :load_test)
|
9
10
|
|
10
11
|
require_relative '../lib/sidekiq/cli'
|
11
12
|
require_relative '../lib/sidekiq/launcher'
|
12
13
|
|
13
14
|
include Sidekiq::Util
|
14
15
|
|
15
|
-
# brew tap shopify/shopify
|
16
|
-
# brew install toxiproxy
|
17
|
-
# gem install toxiproxy
|
18
|
-
require 'toxiproxy'
|
19
|
-
# simulate a non-localhost network for realer-world conditions.
|
20
|
-
# adding 1ms of network latency has an ENORMOUS impact on benchmarks
|
21
|
-
Toxiproxy.populate([{
|
22
|
-
"name": "redis",
|
23
|
-
"listen": "127.0.0.1:6380",
|
24
|
-
"upstream": "127.0.0.1:6379"
|
25
|
-
}])
|
26
|
-
|
27
|
-
|
28
16
|
Sidekiq.configure_server do |config|
|
29
|
-
|
30
|
-
config.redis = {
|
17
|
+
config.options[:concurrency] = 10
|
18
|
+
config.redis = { db: 13, port: 6380, driver: :hiredis }
|
31
19
|
config.options[:queues] << 'default'
|
32
20
|
config.logger.level = Logger::ERROR
|
33
21
|
config.average_scheduled_poll_interval = 2
|
@@ -41,7 +29,8 @@ class LoadWorker
|
|
41
29
|
1
|
42
30
|
end
|
43
31
|
|
44
|
-
def perform(idx)
|
32
|
+
def perform(idx, ts=nil)
|
33
|
+
puts(Time.now.to_f - ts) if ts != nil
|
45
34
|
#raise idx.to_s if idx % 100 == 1
|
46
35
|
end
|
47
36
|
end
|
@@ -49,6 +38,7 @@ end
|
|
49
38
|
# brew tap shopify/shopify
|
50
39
|
# brew install toxiproxy
|
51
40
|
# gem install toxiproxy
|
41
|
+
# run `toxiproxy-server` in a separate terminal window.
|
52
42
|
require 'toxiproxy'
|
53
43
|
# simulate a non-localhost network for realer-world conditions.
|
54
44
|
# adding 1ms of network latency has an ENORMOUS impact on benchmarks
|
@@ -59,7 +49,7 @@ Toxiproxy.populate([{
|
|
59
49
|
}])
|
60
50
|
|
61
51
|
self_read, self_write = IO.pipe
|
62
|
-
%w(INT TERM
|
52
|
+
%w(INT TERM TSTP TTIN).each do |sig|
|
63
53
|
begin
|
64
54
|
trap sig do
|
65
55
|
self_write.puts(sig)
|
@@ -78,19 +68,14 @@ def handle_signal(launcher, sig)
|
|
78
68
|
# http://jira.codehaus.org/browse/JRUBY-4637
|
79
69
|
raise Interrupt
|
80
70
|
when 'TERM'
|
81
|
-
# Heroku sends TERM and then waits
|
71
|
+
# Heroku sends TERM and then waits 30 seconds for process to exit.
|
82
72
|
raise Interrupt
|
83
|
-
when '
|
84
|
-
Sidekiq.logger.info "Received
|
73
|
+
when 'TSTP'
|
74
|
+
Sidekiq.logger.info "Received TSTP, no longer accepting new work"
|
85
75
|
launcher.quiet
|
86
|
-
when 'USR2'
|
87
|
-
if Sidekiq.options[:logfile]
|
88
|
-
Sidekiq.logger.info "Received USR2, reopening log file"
|
89
|
-
Sidekiq::Logging.reopen_logs
|
90
|
-
end
|
91
76
|
when 'TTIN'
|
92
77
|
Thread.list.each do |thread|
|
93
|
-
Sidekiq.logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
|
78
|
+
Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['label']}"
|
94
79
|
if thread.backtrace
|
95
80
|
Sidekiq.logger.warn thread.backtrace.join("\n")
|
96
81
|
else
|
@@ -118,21 +103,26 @@ iter.times do
|
|
118
103
|
end
|
119
104
|
Sidekiq.logger.error "Created #{count*iter} jobs"
|
120
105
|
|
106
|
+
start = Time.now
|
107
|
+
|
121
108
|
Monitoring = Thread.new do
|
122
109
|
watchdog("monitor thread") do
|
123
110
|
while true
|
124
|
-
sleep 2
|
125
|
-
qsize
|
126
|
-
conn.
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
end.map(&:to_i)
|
131
|
-
total = qsize + retries
|
132
|
-
#GC.start
|
133
|
-
Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
|
111
|
+
sleep 0.2
|
112
|
+
qsize = Sidekiq.redis do |conn|
|
113
|
+
conn.llen "queue:default"
|
114
|
+
end
|
115
|
+
total = qsize
|
116
|
+
#Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
|
134
117
|
if total == 0
|
135
|
-
Sidekiq.logger.error("Done")
|
118
|
+
Sidekiq.logger.error("Done, #{iter * count} jobs in #{Time.now - start} sec")
|
119
|
+
Sidekiq.logger.error("Now here's the latency for three jobs")
|
120
|
+
|
121
|
+
LoadWorker.perform_async(1, Time.now.to_f)
|
122
|
+
LoadWorker.perform_async(2, Time.now.to_f)
|
123
|
+
LoadWorker.perform_async(3, Time.now.to_f)
|
124
|
+
|
125
|
+
sleep 0.2
|
136
126
|
exit(0)
|
137
127
|
end
|
138
128
|
end
|
data/bin/sidekiqmon
ADDED
@@ -1,6 +1,6 @@
|
|
1
|
-
|
1
|
+
require 'test_helper'
|
2
2
|
<% module_namespacing do -%>
|
3
|
-
class <%= class_name %>WorkerTest <
|
3
|
+
class <%= class_name %>WorkerTest < Minitest::Test
|
4
4
|
def test_example
|
5
5
|
skip "add some examples to (or delete) #{__FILE__}"
|
6
6
|
end
|
@@ -1,22 +1,24 @@
|
|
1
|
-
require
|
1
|
+
require "rails/generators/named_base"
|
2
2
|
|
3
3
|
module Sidekiq
|
4
4
|
module Generators # :nodoc:
|
5
5
|
class WorkerGenerator < ::Rails::Generators::NamedBase # :nodoc:
|
6
|
-
desc
|
6
|
+
desc "This generator creates a Sidekiq Worker in app/workers and a corresponding test"
|
7
7
|
|
8
|
-
check_class_collision suffix:
|
8
|
+
check_class_collision suffix: "Worker"
|
9
9
|
|
10
10
|
def self.default_generator_root
|
11
11
|
File.dirname(__FILE__)
|
12
12
|
end
|
13
13
|
|
14
14
|
def create_worker_file
|
15
|
-
template
|
15
|
+
template "worker.rb.erb", File.join("app/workers", class_path, "#{file_name}_worker.rb")
|
16
16
|
end
|
17
17
|
|
18
18
|
def create_test_file
|
19
|
-
|
19
|
+
return unless test_framework
|
20
|
+
|
21
|
+
if test_framework == :rspec
|
20
22
|
create_worker_spec
|
21
23
|
else
|
22
24
|
create_worker_test
|
@@ -27,23 +29,29 @@ module Sidekiq
|
|
27
29
|
|
28
30
|
def create_worker_spec
|
29
31
|
template_file = File.join(
|
30
|
-
|
31
|
-
|
32
|
-
|
32
|
+
"spec/workers",
|
33
|
+
class_path,
|
34
|
+
"#{file_name}_worker_spec.rb"
|
33
35
|
)
|
34
|
-
template
|
36
|
+
template "worker_spec.rb.erb", template_file
|
35
37
|
end
|
36
38
|
|
37
39
|
def create_worker_test
|
38
40
|
template_file = File.join(
|
39
|
-
|
40
|
-
|
41
|
-
|
41
|
+
"test/workers",
|
42
|
+
class_path,
|
43
|
+
"#{file_name}_worker_test.rb"
|
42
44
|
)
|
43
|
-
template
|
45
|
+
template "worker_test.rb.erb", template_file
|
44
46
|
end
|
45
47
|
|
48
|
+
def file_name
|
49
|
+
@_file_name ||= super.sub(/_?worker\z/i, "")
|
50
|
+
end
|
46
51
|
|
52
|
+
def test_framework
|
53
|
+
::Rails.application.config.generators.options[:rails][:test_framework]
|
54
|
+
end
|
47
55
|
end
|
48
56
|
end
|
49
57
|
end
|