sidekiq 5.2.8 → 6.0.0

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +0 -2
  3. data/.standard.yml +20 -0
  4. data/6.0-Upgrade.md +70 -0
  5. data/Changes.md +31 -3
  6. data/Ent-2.0-Upgrade.md +37 -0
  7. data/Ent-Changes.md +12 -0
  8. data/Gemfile +12 -11
  9. data/Gemfile.lock +196 -0
  10. data/Pro-5.0-Upgrade.md +25 -0
  11. data/Pro-Changes.md +12 -3
  12. data/README.md +16 -30
  13. data/Rakefile +5 -4
  14. data/bin/sidekiqload +26 -22
  15. data/bin/sidekiqmon +9 -0
  16. data/lib/generators/sidekiq/templates/worker_test.rb.erb +1 -1
  17. data/lib/generators/sidekiq/worker_generator.rb +12 -14
  18. data/lib/sidekiq/api.rb +138 -151
  19. data/lib/sidekiq/cli.rb +97 -162
  20. data/lib/sidekiq/client.rb +45 -46
  21. data/lib/sidekiq/delay.rb +5 -6
  22. data/lib/sidekiq/exception_handler.rb +10 -12
  23. data/lib/sidekiq/extensions/action_mailer.rb +10 -20
  24. data/lib/sidekiq/extensions/active_record.rb +9 -7
  25. data/lib/sidekiq/extensions/class_methods.rb +9 -7
  26. data/lib/sidekiq/extensions/generic_proxy.rb +4 -4
  27. data/lib/sidekiq/fetch.rb +5 -6
  28. data/lib/sidekiq/job_logger.rb +37 -7
  29. data/lib/sidekiq/job_retry.rb +45 -58
  30. data/lib/sidekiq/launcher.rb +59 -51
  31. data/lib/sidekiq/logger.rb +69 -0
  32. data/lib/sidekiq/manager.rb +7 -9
  33. data/lib/sidekiq/middleware/chain.rb +3 -2
  34. data/lib/sidekiq/middleware/i18n.rb +5 -7
  35. data/lib/sidekiq/monitor.rb +148 -0
  36. data/lib/sidekiq/paginator.rb +11 -12
  37. data/lib/sidekiq/processor.rb +52 -49
  38. data/lib/sidekiq/rails.rb +23 -29
  39. data/lib/sidekiq/redis_connection.rb +31 -37
  40. data/lib/sidekiq/scheduled.rb +17 -19
  41. data/lib/sidekiq/testing/inline.rb +2 -1
  42. data/lib/sidekiq/testing.rb +22 -23
  43. data/lib/sidekiq/util.rb +17 -14
  44. data/lib/sidekiq/version.rb +2 -1
  45. data/lib/sidekiq/web/action.rb +14 -10
  46. data/lib/sidekiq/web/application.rb +60 -57
  47. data/lib/sidekiq/web/helpers.rb +66 -67
  48. data/lib/sidekiq/web/router.rb +17 -14
  49. data/lib/sidekiq/web.rb +41 -49
  50. data/lib/sidekiq/worker.rb +124 -97
  51. data/lib/sidekiq.rb +53 -42
  52. data/sidekiq.gemspec +16 -16
  53. data/web/assets/javascripts/dashboard.js +2 -21
  54. data/web/locales/ja.yml +2 -1
  55. metadata +21 -31
  56. data/.travis.yml +0 -11
  57. data/bin/sidekiqctl +0 -20
  58. data/lib/sidekiq/core_ext.rb +0 -1
  59. data/lib/sidekiq/ctl.rb +0 -221
  60. data/lib/sidekiq/logging.rb +0 -122
  61. data/lib/sidekiq/middleware/server/active_record.rb +0 -23
data/bin/sidekiqload CHANGED
@@ -13,8 +13,8 @@ require_relative '../lib/sidekiq/launcher'
13
13
  include Sidekiq::Util
14
14
 
15
15
  Sidekiq.configure_server do |config|
16
- #config.options[:concurrency] = 1
17
- config.redis = { db: 13 }
16
+ config.options[:concurrency] = 10
17
+ config.redis = { db: 13, port: 6380, driver: :hiredis }
18
18
  config.options[:queues] << 'default'
19
19
  config.logger.level = Logger::ERROR
20
20
  config.average_scheduled_poll_interval = 2
@@ -28,7 +28,8 @@ class LoadWorker
28
28
  1
29
29
  end
30
30
 
31
- def perform(idx)
31
+ def perform(idx, ts=nil)
32
+ puts(Time.now.to_f - ts) if ts != nil
32
33
  #raise idx.to_s if idx % 100 == 1
33
34
  end
34
35
  end
@@ -36,14 +37,15 @@ end
36
37
  # brew tap shopify/shopify
37
38
  # brew install toxiproxy
38
39
  # gem install toxiproxy
39
- #require 'toxiproxy'
40
+ # run `toxiproxy-server` in a separate terminal window.
41
+ require 'toxiproxy'
40
42
  # simulate a non-localhost network for realer-world conditions.
41
43
  # adding 1ms of network latency has an ENORMOUS impact on benchmarks
42
- #Toxiproxy.populate([{
43
- #"name": "redis",
44
- #"listen": "127.0.0.1:6380",
45
- #"upstream": "127.0.0.1:6379"
46
- #}])
44
+ Toxiproxy.populate([{
45
+ "name": "redis",
46
+ "listen": "127.0.0.1:6380",
47
+ "upstream": "127.0.0.1:6379"
48
+ }])
47
49
 
48
50
  self_read, self_write = IO.pipe
49
51
  %w(INT TERM TSTP TTIN).each do |sig|
@@ -103,18 +105,20 @@ Sidekiq.logger.error "Created #{count*iter} jobs"
103
105
  Monitoring = Thread.new do
104
106
  watchdog("monitor thread") do
105
107
  while true
106
- sleep 1
107
- qsize, retries = Sidekiq.redis do |conn|
108
- conn.pipelined do
109
- conn.llen "queue:default"
110
- conn.zcard "retry"
111
- end
112
- end.map(&:to_i)
113
- total = qsize + retries
114
- #GC.start
108
+ sleep 0.5
109
+ qsize = Sidekiq.redis do |conn|
110
+ conn.llen "queue:default"
111
+ end
112
+ total = qsize
115
113
  Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
116
114
  if total == 0
117
- Sidekiq.logger.error("Done")
115
+ Sidekiq.logger.error("Done, now here's the latency for three jobs")
116
+
117
+ LoadWorker.perform_async(1, Time.now.to_f)
118
+ LoadWorker.perform_async(2, Time.now.to_f)
119
+ LoadWorker.perform_async(3, Time.now.to_f)
120
+
121
+ sleep 0.2
118
122
  exit(0)
119
123
  end
120
124
  end
@@ -125,8 +129,8 @@ begin
125
129
  #RubyProf::exclude_threads = [ Monitoring ]
126
130
  #RubyProf.start
127
131
  fire_event(:startup)
128
- #Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
129
- #Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
132
+ Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
133
+ Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
130
134
  launcher = Sidekiq::Launcher.new(Sidekiq.options)
131
135
  launcher.run
132
136
 
@@ -134,7 +138,7 @@ begin
134
138
  signal = readable_io.first[0].gets.strip
135
139
  handle_signal(launcher, signal)
136
140
  end
137
- #end
141
+ end
138
142
  rescue SystemExit => e
139
143
  #Sidekiq.logger.error("Profiling...")
140
144
  #result = RubyProf.stop
data/bin/sidekiqmon ADDED
@@ -0,0 +1,9 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'sidekiq/monitor'
4
+
5
+ if ARGV[0] == 'status'
6
+ Sidekiq::Monitor::Status.new.display(ARGV[1])
7
+ else
8
+ Sidekiq::Monitor.print_usage
9
+ end
@@ -1,6 +1,6 @@
1
1
  require 'test_helper'
2
2
  <% module_namespacing do -%>
3
- class <%= class_name %>WorkerTest < <% if defined? Minitest::Test %>Minitest::Test<% else %>MiniTest::Unit::TestCase<% end %>
3
+ class <%= class_name %>WorkerTest < Minitest::Test
4
4
  def test_example
5
5
  skip "add some examples to (or delete) #{__FILE__}"
6
6
  end
@@ -1,18 +1,18 @@
1
- require 'rails/generators/named_base'
1
+ require "rails/generators/named_base"
2
2
 
3
3
  module Sidekiq
4
4
  module Generators # :nodoc:
5
5
  class WorkerGenerator < ::Rails::Generators::NamedBase # :nodoc:
6
- desc 'This generator creates a Sidekiq Worker in app/workers and a corresponding test'
6
+ desc "This generator creates a Sidekiq Worker in app/workers and a corresponding test"
7
7
 
8
- check_class_collision suffix: 'Worker'
8
+ check_class_collision suffix: "Worker"
9
9
 
10
10
  def self.default_generator_root
11
11
  File.dirname(__FILE__)
12
12
  end
13
13
 
14
14
  def create_worker_file
15
- template 'worker.rb.erb', File.join('app/workers', class_path, "#{file_name}_worker.rb")
15
+ template "worker.rb.erb", File.join("app/workers", class_path, "#{file_name}_worker.rb")
16
16
  end
17
17
 
18
18
  def create_test_file
@@ -27,23 +27,21 @@ module Sidekiq
27
27
 
28
28
  def create_worker_spec
29
29
  template_file = File.join(
30
- 'spec/workers',
31
- class_path,
32
- "#{file_name}_worker_spec.rb"
30
+ "spec/workers",
31
+ class_path,
32
+ "#{file_name}_worker_spec.rb"
33
33
  )
34
- template 'worker_spec.rb.erb', template_file
34
+ template "worker_spec.rb.erb", template_file
35
35
  end
36
36
 
37
37
  def create_worker_test
38
38
  template_file = File.join(
39
- 'test/workers',
40
- class_path,
41
- "#{file_name}_worker_test.rb"
39
+ "test/workers",
40
+ class_path,
41
+ "#{file_name}_worker_test.rb"
42
42
  )
43
- template 'worker_test.rb.erb', template_file
43
+ template "worker_test.rb.erb", template_file
44
44
  end
45
-
46
-
47
45
  end
48
46
  end
49
47
  end