sr-sidekiq 4.1.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (186) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +12 -0
  3. data/3.0-Upgrade.md +70 -0
  4. data/4.0-Upgrade.md +50 -0
  5. data/COMM-LICENSE (sidekiq) +95 -0
  6. data/Changes.md +1241 -0
  7. data/Ent-Changes.md +112 -0
  8. data/Gemfile +29 -0
  9. data/LICENSE (sidekiq) +9 -0
  10. data/LICENSE (sr-sidekiq) +5 -0
  11. data/Pro-2.0-Upgrade.md +138 -0
  12. data/Pro-3.0-Upgrade.md +44 -0
  13. data/Pro-Changes.md +539 -0
  14. data/README.md +8 -0
  15. data/Rakefile +9 -0
  16. data/bin/sidekiq +18 -0
  17. data/bin/sidekiqctl +99 -0
  18. data/bin/sidekiqload +167 -0
  19. data/code_of_conduct.md +50 -0
  20. data/lib/generators/sidekiq/templates/worker.rb.erb +9 -0
  21. data/lib/generators/sidekiq/templates/worker_spec.rb.erb +6 -0
  22. data/lib/generators/sidekiq/templates/worker_test.rb.erb +8 -0
  23. data/lib/generators/sidekiq/worker_generator.rb +49 -0
  24. data/lib/sidekiq.rb +237 -0
  25. data/lib/sidekiq/api.rb +844 -0
  26. data/lib/sidekiq/cli.rb +389 -0
  27. data/lib/sidekiq/client.rb +260 -0
  28. data/lib/sidekiq/core_ext.rb +106 -0
  29. data/lib/sidekiq/exception_handler.rb +31 -0
  30. data/lib/sidekiq/extensions/action_mailer.rb +57 -0
  31. data/lib/sidekiq/extensions/active_record.rb +40 -0
  32. data/lib/sidekiq/extensions/class_methods.rb +40 -0
  33. data/lib/sidekiq/extensions/generic_proxy.rb +25 -0
  34. data/lib/sidekiq/fetch.rb +81 -0
  35. data/lib/sidekiq/launcher.rb +160 -0
  36. data/lib/sidekiq/logging.rb +106 -0
  37. data/lib/sidekiq/manager.rb +137 -0
  38. data/lib/sidekiq/middleware/chain.rb +150 -0
  39. data/lib/sidekiq/middleware/i18n.rb +42 -0
  40. data/lib/sidekiq/middleware/server/active_record.rb +13 -0
  41. data/lib/sidekiq/middleware/server/logging.rb +40 -0
  42. data/lib/sidekiq/middleware/server/retry_jobs.rb +205 -0
  43. data/lib/sidekiq/paginator.rb +43 -0
  44. data/lib/sidekiq/processor.rb +186 -0
  45. data/lib/sidekiq/rails.rb +39 -0
  46. data/lib/sidekiq/redis_connection.rb +97 -0
  47. data/lib/sidekiq/scheduled.rb +146 -0
  48. data/lib/sidekiq/testing.rb +316 -0
  49. data/lib/sidekiq/testing/inline.rb +29 -0
  50. data/lib/sidekiq/util.rb +62 -0
  51. data/lib/sidekiq/version.rb +4 -0
  52. data/lib/sidekiq/web.rb +278 -0
  53. data/lib/sidekiq/web_helpers.rb +255 -0
  54. data/lib/sidekiq/worker.rb +121 -0
  55. data/sidekiq.gemspec +26 -0
  56. data/sr-sidekiq-4.1.3.gem +0 -0
  57. data/sr-sidekiq-4.1.4.gem +0 -0
  58. data/sr-sidekiq-4.1.5.gem +0 -0
  59. data/test/config.yml +9 -0
  60. data/test/env_based_config.yml +11 -0
  61. data/test/fake_env.rb +1 -0
  62. data/test/fixtures/en.yml +2 -0
  63. data/test/helper.rb +75 -0
  64. data/test/test_actors.rb +138 -0
  65. data/test/test_api.rb +528 -0
  66. data/test/test_cli.rb +406 -0
  67. data/test/test_client.rb +262 -0
  68. data/test/test_exception_handler.rb +56 -0
  69. data/test/test_extensions.rb +127 -0
  70. data/test/test_fetch.rb +50 -0
  71. data/test/test_launcher.rb +85 -0
  72. data/test/test_logging.rb +35 -0
  73. data/test/test_manager.rb +50 -0
  74. data/test/test_middleware.rb +158 -0
  75. data/test/test_processor.rb +201 -0
  76. data/test/test_rails.rb +22 -0
  77. data/test/test_redis_connection.rb +127 -0
  78. data/test/test_retry.rb +326 -0
  79. data/test/test_retry_exhausted.rb +149 -0
  80. data/test/test_scheduled.rb +115 -0
  81. data/test/test_scheduling.rb +50 -0
  82. data/test/test_sidekiq.rb +107 -0
  83. data/test/test_testing.rb +143 -0
  84. data/test/test_testing_fake.rb +357 -0
  85. data/test/test_testing_inline.rb +94 -0
  86. data/test/test_util.rb +13 -0
  87. data/test/test_web.rb +614 -0
  88. data/test/test_web_helpers.rb +54 -0
  89. data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
  90. data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
  91. data/web/assets/images/favicon.ico +0 -0
  92. data/web/assets/images/logo.png +0 -0
  93. data/web/assets/images/status-sd8051fd480.png +0 -0
  94. data/web/assets/images/status/active.png +0 -0
  95. data/web/assets/images/status/idle.png +0 -0
  96. data/web/assets/javascripts/application.js +88 -0
  97. data/web/assets/javascripts/dashboard.js +300 -0
  98. data/web/assets/javascripts/locales/README.md +27 -0
  99. data/web/assets/javascripts/locales/jquery.timeago.ar.js +96 -0
  100. data/web/assets/javascripts/locales/jquery.timeago.bg.js +18 -0
  101. data/web/assets/javascripts/locales/jquery.timeago.bs.js +49 -0
  102. data/web/assets/javascripts/locales/jquery.timeago.ca.js +18 -0
  103. data/web/assets/javascripts/locales/jquery.timeago.cs.js +18 -0
  104. data/web/assets/javascripts/locales/jquery.timeago.cy.js +20 -0
  105. data/web/assets/javascripts/locales/jquery.timeago.da.js +18 -0
  106. data/web/assets/javascripts/locales/jquery.timeago.de.js +18 -0
  107. data/web/assets/javascripts/locales/jquery.timeago.el.js +18 -0
  108. data/web/assets/javascripts/locales/jquery.timeago.en-short.js +20 -0
  109. data/web/assets/javascripts/locales/jquery.timeago.en.js +20 -0
  110. data/web/assets/javascripts/locales/jquery.timeago.es.js +18 -0
  111. data/web/assets/javascripts/locales/jquery.timeago.et.js +18 -0
  112. data/web/assets/javascripts/locales/jquery.timeago.fa.js +22 -0
  113. data/web/assets/javascripts/locales/jquery.timeago.fi.js +28 -0
  114. data/web/assets/javascripts/locales/jquery.timeago.fr-short.js +16 -0
  115. data/web/assets/javascripts/locales/jquery.timeago.fr.js +17 -0
  116. data/web/assets/javascripts/locales/jquery.timeago.he.js +18 -0
  117. data/web/assets/javascripts/locales/jquery.timeago.hr.js +49 -0
  118. data/web/assets/javascripts/locales/jquery.timeago.hu.js +18 -0
  119. data/web/assets/javascripts/locales/jquery.timeago.hy.js +18 -0
  120. data/web/assets/javascripts/locales/jquery.timeago.id.js +18 -0
  121. data/web/assets/javascripts/locales/jquery.timeago.it.js +16 -0
  122. data/web/assets/javascripts/locales/jquery.timeago.ja.js +19 -0
  123. data/web/assets/javascripts/locales/jquery.timeago.ko.js +17 -0
  124. data/web/assets/javascripts/locales/jquery.timeago.lt.js +20 -0
  125. data/web/assets/javascripts/locales/jquery.timeago.mk.js +20 -0
  126. data/web/assets/javascripts/locales/jquery.timeago.nl.js +20 -0
  127. data/web/assets/javascripts/locales/jquery.timeago.no.js +18 -0
  128. data/web/assets/javascripts/locales/jquery.timeago.pl.js +31 -0
  129. data/web/assets/javascripts/locales/jquery.timeago.pt-br.js +16 -0
  130. data/web/assets/javascripts/locales/jquery.timeago.pt.js +16 -0
  131. data/web/assets/javascripts/locales/jquery.timeago.ro.js +18 -0
  132. data/web/assets/javascripts/locales/jquery.timeago.rs.js +49 -0
  133. data/web/assets/javascripts/locales/jquery.timeago.ru.js +34 -0
  134. data/web/assets/javascripts/locales/jquery.timeago.sk.js +18 -0
  135. data/web/assets/javascripts/locales/jquery.timeago.sl.js +44 -0
  136. data/web/assets/javascripts/locales/jquery.timeago.sv.js +18 -0
  137. data/web/assets/javascripts/locales/jquery.timeago.th.js +20 -0
  138. data/web/assets/javascripts/locales/jquery.timeago.tr.js +16 -0
  139. data/web/assets/javascripts/locales/jquery.timeago.uk.js +34 -0
  140. data/web/assets/javascripts/locales/jquery.timeago.uz.js +19 -0
  141. data/web/assets/javascripts/locales/jquery.timeago.zh-cn.js +20 -0
  142. data/web/assets/javascripts/locales/jquery.timeago.zh-tw.js +20 -0
  143. data/web/assets/stylesheets/application.css +754 -0
  144. data/web/assets/stylesheets/bootstrap.css +9 -0
  145. data/web/locales/cs.yml +78 -0
  146. data/web/locales/da.yml +68 -0
  147. data/web/locales/de.yml +69 -0
  148. data/web/locales/el.yml +68 -0
  149. data/web/locales/en.yml +79 -0
  150. data/web/locales/es.yml +69 -0
  151. data/web/locales/fr.yml +78 -0
  152. data/web/locales/hi.yml +75 -0
  153. data/web/locales/it.yml +69 -0
  154. data/web/locales/ja.yml +78 -0
  155. data/web/locales/ko.yml +68 -0
  156. data/web/locales/nb.yml +77 -0
  157. data/web/locales/nl.yml +68 -0
  158. data/web/locales/pl.yml +59 -0
  159. data/web/locales/pt-br.yml +68 -0
  160. data/web/locales/pt.yml +67 -0
  161. data/web/locales/ru.yml +78 -0
  162. data/web/locales/sv.yml +68 -0
  163. data/web/locales/ta.yml +75 -0
  164. data/web/locales/uk.yml +76 -0
  165. data/web/locales/zh-cn.yml +68 -0
  166. data/web/locales/zh-tw.yml +68 -0
  167. data/web/views/_footer.erb +17 -0
  168. data/web/views/_job_info.erb +88 -0
  169. data/web/views/_nav.erb +66 -0
  170. data/web/views/_paging.erb +23 -0
  171. data/web/views/_poll_js.erb +5 -0
  172. data/web/views/_poll_link.erb +7 -0
  173. data/web/views/_status.erb +4 -0
  174. data/web/views/_summary.erb +40 -0
  175. data/web/views/busy.erb +94 -0
  176. data/web/views/dashboard.erb +75 -0
  177. data/web/views/dead.erb +34 -0
  178. data/web/views/layout.erb +32 -0
  179. data/web/views/morgue.erb +71 -0
  180. data/web/views/queue.erb +45 -0
  181. data/web/views/queues.erb +28 -0
  182. data/web/views/retries.erb +74 -0
  183. data/web/views/retry.erb +34 -0
  184. data/web/views/scheduled.erb +54 -0
  185. data/web/views/scheduled_job_info.erb +8 -0
  186. metadata +408 -0
@@ -0,0 +1,121 @@
1
+ # frozen_string_literal: true
2
+ require 'sidekiq/client'
3
+ require 'sidekiq/core_ext'
4
+
5
+ module Sidekiq
6
+
7
+ ##
8
+ # Include this module in your worker class and you can easily create
9
+ # asynchronous jobs:
10
+ #
11
+ # class HardWorker
12
+ # include Sidekiq::Worker
13
+ #
14
+ # def perform(*args)
15
+ # # do some work
16
+ # end
17
+ # end
18
+ #
19
+ # Then in your Rails app, you can do this:
20
+ #
21
+ # HardWorker.perform_async(1, 2, 3)
22
+ #
23
+ # Note that perform_async is a class method, perform is an instance method.
24
+ module Worker
25
+ attr_accessor :jid
26
+
27
+ def self.included(base)
28
+ raise ArgumentError, "You cannot include Sidekiq::Worker in an ActiveJob: #{base.name}" if base.ancestors.any? {|c| c.name == 'ActiveJob::Base' }
29
+
30
+ base.extend(ClassMethods)
31
+ base.class_attribute :sidekiq_options_hash
32
+ base.class_attribute :sidekiq_retry_in_block
33
+ base.class_attribute :sidekiq_retries_exhausted_block
34
+ end
35
+
36
+ def logger
37
+ Sidekiq.logger
38
+ end
39
+
40
+ module ClassMethods
41
+
42
+ def delay(*args)
43
+ raise ArgumentError, "Do not call .delay on a Sidekiq::Worker class, call .perform_async"
44
+ end
45
+
46
+ def delay_for(*args)
47
+ raise ArgumentError, "Do not call .delay_for on a Sidekiq::Worker class, call .perform_in"
48
+ end
49
+
50
+ def delay_until(*args)
51
+ raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
52
+ end
53
+
54
+ def set(options)
55
+ Thread.current[:sidekiq_worker_set] = options
56
+ self
57
+ end
58
+
59
+ def perform_async(*args)
60
+ client_push('class' => self, 'args' => args)
61
+ end
62
+
63
+ # +interval+ must be a timestamp, numeric or something that acts
64
+ # numeric (like an activesupport time interval).
65
+ def perform_in(interval, *args)
66
+ int = interval.to_f
67
+ now = Time.now
68
+ ts = (int < 1_000_000_000 ? (now + interval).to_f : int)
69
+
70
+ item = { 'class' => self, 'args' => args, 'at' => ts }
71
+
72
+ # Optimization to enqueue something now that is scheduled to go out now or in the past
73
+ item.delete('at'.freeze) if ts <= now.to_f
74
+
75
+ client_push(item)
76
+ end
77
+ alias_method :perform_at, :perform_in
78
+
79
+ ##
80
+ # Allows customization for this type of Worker.
81
+ # Legal options:
82
+ #
83
+ # queue - use a named queue for this Worker, default 'default'
84
+ # retry - enable the RetryJobs middleware for this Worker, *true* to use the default
85
+ # or *Integer* count
86
+ # backtrace - whether to save any error backtrace in the retry payload to display in web UI,
87
+ # can be true, false or an integer number of lines to save, default *false*
88
+ # pool - use the given Redis connection pool to push this type of job to a given shard.
89
+ #
90
+ # In practice, any option is allowed. This is the main mechanism to configure the
91
+ # options for a specific job.
92
+ def sidekiq_options(opts={})
93
+ self.sidekiq_options_hash = get_sidekiq_options.merge(opts.stringify_keys)
94
+ end
95
+
96
+ def sidekiq_retry_in(&block)
97
+ self.sidekiq_retry_in_block = block
98
+ end
99
+
100
+ def sidekiq_retries_exhausted(&block)
101
+ self.sidekiq_retries_exhausted_block = block
102
+ end
103
+
104
+ def get_sidekiq_options # :nodoc:
105
+ self.sidekiq_options_hash ||= Sidekiq.default_worker_options
106
+ end
107
+
108
+ def client_push(item) # :nodoc:
109
+ pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options['pool'] || Sidekiq.redis_pool
110
+ hash = if Thread.current[:sidekiq_worker_set]
111
+ x, Thread.current[:sidekiq_worker_set] = Thread.current[:sidekiq_worker_set], nil
112
+ x.stringify_keys.merge(item.stringify_keys)
113
+ else
114
+ item.stringify_keys
115
+ end
116
+ Sidekiq::Client.new(pool).push(hash)
117
+ end
118
+
119
+ end
120
+ end
121
+ end
data/sidekiq.gemspec ADDED
@@ -0,0 +1,26 @@
1
+ # -*- encoding: utf-8 -*-
2
+ require File.expand_path('../lib/sidekiq/version', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.authors = ["Sean Huber"]
6
+ gem.email = ["seanhuber@seanhuber.com"]
7
+ gem.summary = "A fork of Sidekiq (v4.1.2) with worker parameter restrictions"
8
+ gem.description = "A fork of Sidekiq (v4.1.2) with worker parameter restrictions"
9
+ gem.homepage = "https://github.com/seanhuber/sr-sidekiq"
10
+ gem.license = "LGPL-3.0"
11
+
12
+ gem.executables = ['sidekiq', 'sidekiqctl']
13
+ gem.files = `git ls-files | grep -Ev '^(myapp|examples)'`.split("\n")
14
+ gem.test_files = `git ls-files -- test/*`.split("\n")
15
+ gem.name = "sr-sidekiq"
16
+ gem.require_paths = ["lib"]
17
+ gem.version = Sidekiq::VERSION
18
+ gem.add_dependency 'redis', '~> 3.2', '>= 3.2.1'
19
+ gem.add_dependency 'connection_pool', '~> 2.2', '>= 2.2.0'
20
+ gem.add_dependency 'concurrent-ruby', '~> 1.0'
21
+ gem.add_development_dependency 'redis-namespace', '~> 1.5', '>= 1.5.2'
22
+ gem.add_development_dependency 'sinatra', '~> 1.4', '>= 1.4.6'
23
+ gem.add_development_dependency 'minitest', '~> 5.7', '>= 5.7.0'
24
+ gem.add_development_dependency 'rake', '~> 10.0'
25
+ gem.add_development_dependency 'rails', '~> 4', '>= 3.2.0'
26
+ end
Binary file
Binary file
Binary file
data/test/config.yml ADDED
@@ -0,0 +1,9 @@
1
+ ---
2
+ :verbose: false
3
+ :require: ./test/fake_env.rb
4
+ :pidfile: /tmp/sidekiq-config-test.pid
5
+ :logfile: /tmp/sidekiq.log
6
+ :concurrency: 50
7
+ :queues:
8
+ - [<%="very_"%>often, 2]
9
+ - [seldom, 1]
@@ -0,0 +1,11 @@
1
+ ---
2
+ :pidfile: /tmp/sidekiq-config-test.pid
3
+ :concurrency: 50
4
+ staging:
5
+ :verbose: false
6
+ :require: ./test/fake_env.rb
7
+ :logfile: /tmp/sidekiq.log
8
+ :concurrency: 5
9
+ :queues:
10
+ - [<%="very_"%>often, 2]
11
+ - [seldom, 1]
data/test/fake_env.rb ADDED
@@ -0,0 +1 @@
1
+ # frozen_string_literal: true
@@ -0,0 +1,2 @@
1
+ en:
2
+ translated_text: 'Changed text from add locals'
data/test/helper.rb ADDED
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+ $TESTING = true
3
+ # disable minitest/parallel threads
4
+ ENV["N"] = "0"
5
+
6
+ if ENV["COVERAGE"]
7
+ require 'simplecov'
8
+ SimpleCov.start do
9
+ add_filter "/test/"
10
+ add_filter "/myapp/"
11
+ end
12
+ end
13
+ ENV['RACK_ENV'] = ENV['RAILS_ENV'] = 'test'
14
+
15
+ trap 'USR1' do
16
+ threads = Thread.list
17
+
18
+ puts
19
+ puts "=" * 80
20
+ puts "Received USR1 signal; printing all #{threads.count} thread backtraces."
21
+
22
+ threads.each do |thr|
23
+ description = thr == Thread.main ? "Main thread" : thr.inspect
24
+ puts
25
+ puts "#{description} backtrace: "
26
+ puts thr.backtrace.join("\n")
27
+ end
28
+
29
+ puts "=" * 80
30
+ end
31
+
32
+ begin
33
+ require 'pry-byebug'
34
+ rescue LoadError
35
+ end
36
+
37
+ require 'minitest/autorun'
38
+
39
+ require 'sidekiq'
40
+ require 'sidekiq/util'
41
+ Sidekiq.logger.level = Logger::ERROR
42
+
43
+ Sidekiq::Test = Minitest::Test
44
+
45
+ require 'sidekiq/redis_connection'
46
+ REDIS_URL = ENV['REDIS_URL'] || 'redis://localhost/15'
47
+ REDIS = Sidekiq::RedisConnection.create(:url => REDIS_URL, :namespace => 'testy')
48
+
49
+ Sidekiq.configure_client do |config|
50
+ config.redis = { :url => REDIS_URL, :namespace => 'testy' }
51
+ end
52
+
53
+ def capture_logging(lvl=Logger::INFO)
54
+ old = Sidekiq.logger
55
+ begin
56
+ out = StringIO.new
57
+ logger = Logger.new(out)
58
+ logger.level = lvl
59
+ Sidekiq.logger = logger
60
+ yield
61
+ out.string
62
+ ensure
63
+ Sidekiq.logger = old
64
+ end
65
+ end
66
+
67
+ def with_logging(lvl=Logger::DEBUG)
68
+ old = Sidekiq.logger.level
69
+ begin
70
+ Sidekiq.logger.level = lvl
71
+ yield
72
+ ensure
73
+ Sidekiq.logger.level = old
74
+ end
75
+ end
@@ -0,0 +1,138 @@
1
+ # frozen_string_literal: true
2
+ require_relative 'helper'
3
+ require 'sidekiq/cli'
4
+ require 'sidekiq/fetch'
5
+ require 'sidekiq/scheduled'
6
+ require 'sidekiq/processor'
7
+
8
+ class TestActors < Sidekiq::Test
9
+ class JoeWorker
10
+ include Sidekiq::Worker
11
+ def perform(slp)
12
+ raise "boom" if slp == "boom"
13
+ sleep(slp) if slp > 0
14
+ $count += 1
15
+ end
16
+ end
17
+
18
+ describe 'threads' do
19
+ before do
20
+ Sidekiq.redis {|c| c.flushdb}
21
+ end
22
+
23
+ describe 'scheduler' do
24
+ it 'can start and stop' do
25
+ f = Sidekiq::Scheduled::Poller.new
26
+ f.start
27
+ f.terminate
28
+ end
29
+
30
+ it 'can schedule' do
31
+ ss = Sidekiq::ScheduledSet.new
32
+ q = Sidekiq::Queue.new
33
+
34
+ JoeWorker.perform_in(0.01, 0)
35
+
36
+ assert_equal 0, q.size
37
+ assert_equal 1, ss.size
38
+
39
+ sleep 0.015
40
+ s = Sidekiq::Scheduled::Poller.new
41
+ s.enqueue
42
+ assert_equal 1, q.size
43
+ assert_equal 0, ss.size
44
+ s.terminate
45
+ end
46
+ end
47
+
48
+ describe 'processor' do
49
+ before do
50
+ $count = 0
51
+ end
52
+
53
+ it 'can start and stop' do
54
+ f = Sidekiq::Processor.new(Mgr.new)
55
+ f.terminate
56
+ end
57
+
58
+ class Mgr
59
+ attr_reader :latest_error
60
+ attr_reader :mutex
61
+ attr_reader :cond
62
+ def initialize
63
+ @mutex = ::Mutex.new
64
+ @cond = ::ConditionVariable.new
65
+ end
66
+ def processor_died(inst, err)
67
+ @latest_error = err
68
+ @mutex.synchronize do
69
+ @cond.signal
70
+ end
71
+ end
72
+ def processor_stopped(inst)
73
+ @mutex.synchronize do
74
+ @cond.signal
75
+ end
76
+ end
77
+ def options
78
+ { :concurrency => 3, :queues => ['default'] }
79
+ end
80
+ end
81
+
82
+ it 'can process' do
83
+ mgr = Mgr.new
84
+
85
+ p = Sidekiq::Processor.new(mgr)
86
+ JoeWorker.perform_async(0)
87
+
88
+ a = $count
89
+ p.process_one
90
+ b = $count
91
+ assert_equal a + 1, b
92
+ end
93
+
94
+ it 'deals with errors' do
95
+ mgr = Mgr.new
96
+
97
+ p = Sidekiq::Processor.new(mgr)
98
+ JoeWorker.perform_async("boom")
99
+ q = Sidekiq::Queue.new
100
+ assert_equal 1, q.size
101
+
102
+ a = $count
103
+ mgr.mutex.synchronize do
104
+ p.start
105
+ mgr.cond.wait(mgr.mutex)
106
+ end
107
+ b = $count
108
+ assert_equal a, b
109
+
110
+ sleep 0.001
111
+ assert_equal false, p.thread.status
112
+ p.terminate(true)
113
+ refute_nil mgr.latest_error
114
+ assert_equal RuntimeError, mgr.latest_error.class
115
+ end
116
+
117
+ it 'gracefully kills' do
118
+ mgr = Mgr.new
119
+
120
+ p = Sidekiq::Processor.new(mgr)
121
+ JoeWorker.perform_async(1)
122
+ q = Sidekiq::Queue.new
123
+ assert_equal 1, q.size
124
+
125
+ a = $count
126
+ p.start
127
+ sleep(0.02)
128
+ p.terminate
129
+ p.kill(true)
130
+
131
+ b = $count
132
+ assert_equal a, b
133
+ assert_equal false, p.thread.status
134
+ refute mgr.latest_error, mgr.latest_error.to_s
135
+ end
136
+ end
137
+ end
138
+ end
data/test/test_api.rb ADDED
@@ -0,0 +1,528 @@
1
+ # frozen_string_literal: true
2
+ require_relative 'helper'
3
+ require 'sidekiq/api'
4
+ require 'active_job'
5
+ require 'action_mailer'
6
+
7
+ class TestApi < Sidekiq::Test
8
+ describe 'api' do
9
+ before do
10
+ Sidekiq.redis {|c| c.flushdb }
11
+ end
12
+
13
+ describe "stats" do
14
+ it "is initially zero" do
15
+ s = Sidekiq::Stats.new
16
+ assert_equal 0, s.processed
17
+ assert_equal 0, s.failed
18
+ assert_equal 0, s.enqueued
19
+ end
20
+
21
+ describe "processed" do
22
+ it "returns number of processed jobs" do
23
+ Sidekiq.redis { |conn| conn.set("stat:processed", 5) }
24
+ s = Sidekiq::Stats.new
25
+ assert_equal 5, s.processed
26
+ end
27
+ end
28
+
29
+ describe "failed" do
30
+ it "returns number of failed jobs" do
31
+ Sidekiq.redis { |conn| conn.set("stat:failed", 5) }
32
+ s = Sidekiq::Stats.new
33
+ assert_equal 5, s.failed
34
+ end
35
+ end
36
+
37
+ describe "reset" do
38
+ before do
39
+ Sidekiq.redis do |conn|
40
+ conn.set('stat:processed', 5)
41
+ conn.set('stat:failed', 10)
42
+ end
43
+ end
44
+
45
+ it 'will reset all stats by default' do
46
+ Sidekiq::Stats.new.reset
47
+ s = Sidekiq::Stats.new
48
+ assert_equal 0, s.failed
49
+ assert_equal 0, s.processed
50
+ end
51
+
52
+ it 'can reset individual stats' do
53
+ Sidekiq::Stats.new.reset('failed')
54
+ s = Sidekiq::Stats.new
55
+ assert_equal 0, s.failed
56
+ assert_equal 5, s.processed
57
+ end
58
+
59
+ it 'can accept anything that responds to #to_s' do
60
+ Sidekiq::Stats.new.reset(:failed)
61
+ s = Sidekiq::Stats.new
62
+ assert_equal 0, s.failed
63
+ assert_equal 5, s.processed
64
+ end
65
+
66
+ it 'ignores anything other than "failed" or "processed"' do
67
+ Sidekiq::Stats.new.reset((1..10).to_a, ['failed'])
68
+ s = Sidekiq::Stats.new
69
+ assert_equal 0, s.failed
70
+ assert_equal 5, s.processed
71
+ end
72
+ end
73
+
74
+ describe "queues" do
75
+ it "is initially empty" do
76
+ s = Sidekiq::Stats::Queues.new
77
+ assert_equal 0, s.lengths.size
78
+ end
79
+
80
+ it "returns a hash of queue and size in order" do
81
+ Sidekiq.redis do |conn|
82
+ conn.rpush 'queue:foo', '{}'
83
+ conn.sadd 'queues', 'foo'
84
+
85
+ 3.times { conn.rpush 'queue:bar', '{}' }
86
+ conn.sadd 'queues', 'bar'
87
+ end
88
+
89
+ s = Sidekiq::Stats::Queues.new
90
+ assert_equal ({ "foo" => 1, "bar" => 3 }), s.lengths
91
+ assert_equal "bar", s.lengths.first.first
92
+
93
+ assert_equal Sidekiq::Stats.new.queues, Sidekiq::Stats::Queues.new.lengths
94
+ end
95
+ end
96
+
97
+ describe "enqueued" do
98
+ it "returns total enqueued jobs" do
99
+ Sidekiq.redis do |conn|
100
+ conn.rpush 'queue:foo', '{}'
101
+ conn.sadd 'queues', 'foo'
102
+
103
+ 3.times { conn.rpush 'queue:bar', '{}' }
104
+ conn.sadd 'queues', 'bar'
105
+ end
106
+
107
+ s = Sidekiq::Stats.new
108
+ assert_equal 4, s.enqueued
109
+ end
110
+ end
111
+
112
+ describe "over time" do
113
+ before do
114
+ require 'active_support/core_ext/time/conversions'
115
+ @before = Time::DATE_FORMATS[:default]
116
+ Time::DATE_FORMATS[:default] = "%d/%m/%Y %H:%M:%S"
117
+ end
118
+
119
+ after do
120
+ Time::DATE_FORMATS[:default] = @before
121
+ end
122
+
123
+ describe "processed" do
124
+ it 'retrieves hash of dates' do
125
+ Sidekiq.redis do |c|
126
+ c.incrby("stat:processed:2012-12-24", 4)
127
+ c.incrby("stat:processed:2012-12-25", 1)
128
+ c.incrby("stat:processed:2012-12-26", 6)
129
+ c.incrby("stat:processed:2012-12-27", 2)
130
+ end
131
+ Time.stub(:now, Time.parse("2012-12-26 1:00:00 -0500")) do
132
+ s = Sidekiq::Stats::History.new(2)
133
+ assert_equal({ "2012-12-26" => 6, "2012-12-25" => 1 }, s.processed)
134
+
135
+ s = Sidekiq::Stats::History.new(3)
136
+ assert_equal({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }, s.processed)
137
+
138
+ s = Sidekiq::Stats::History.new(2, Date.parse("2012-12-25"))
139
+ assert_equal({ "2012-12-25" => 1, "2012-12-24" => 4 }, s.processed)
140
+ end
141
+ end
142
+ end
143
+
144
+ describe "failed" do
145
+ it 'retrieves hash of dates' do
146
+ Sidekiq.redis do |c|
147
+ c.incrby("stat:failed:2012-12-24", 4)
148
+ c.incrby("stat:failed:2012-12-25", 1)
149
+ c.incrby("stat:failed:2012-12-26", 6)
150
+ c.incrby("stat:failed:2012-12-27", 2)
151
+ end
152
+ Time.stub(:now, Time.parse("2012-12-26 1:00:00 -0500")) do
153
+ s = Sidekiq::Stats::History.new(2)
154
+ assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1 }), s.failed
155
+
156
+ s = Sidekiq::Stats::History.new(3)
157
+ assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }), s.failed
158
+
159
+ s = Sidekiq::Stats::History.new(2, Date.parse("2012-12-25"))
160
+ assert_equal ({ "2012-12-25" => 1, "2012-12-24" => 4 }), s.failed
161
+ end
162
+ end
163
+ end
164
+ end
165
+ end
166
+
167
+ describe 'with an empty database' do
168
+ it 'shows queue as empty' do
169
+ q = Sidekiq::Queue.new
170
+ assert_equal 0, q.size
171
+ assert_equal 0, q.latency
172
+ end
173
+
174
+ before do
175
+ ActiveJob::Base.queue_adapter = :sidekiq
176
+ ActiveJob::Base.logger = nil
177
+
178
+ class ApiMailer < ActionMailer::Base
179
+ def test_email(*)
180
+ end
181
+ end
182
+
183
+ class ApiJob < ActiveJob::Base
184
+ def perform(*)
185
+ end
186
+ end
187
+ end
188
+
189
+ class ApiWorker
190
+ include Sidekiq::Worker
191
+ end
192
+
193
+ it 'can enumerate jobs' do
194
+ q = Sidekiq::Queue.new
195
+ Time.stub(:now, Time.new(2012, 12, 26)) do
196
+ ApiWorker.perform_async(1, 'mike')
197
+ assert_equal ['TestApi::ApiWorker'], q.map(&:klass)
198
+
199
+ job = q.first
200
+ assert_equal 24, job.jid.size
201
+ assert_equal [1, 'mike'], job.args
202
+ assert_equal Time.new(2012, 12, 26), job.enqueued_at
203
+ end
204
+ assert q.latency > 10_000_000
205
+
206
+ q = Sidekiq::Queue.new('other')
207
+ assert_equal 0, q.size
208
+ end
209
+
210
+ it 'has no enqueued_at time for jobs enqueued in the future' do
211
+ job_id = ApiWorker.perform_in(100, 1, 'foo')
212
+ job = Sidekiq::ScheduledSet.new.find_job(job_id)
213
+ assert_nil job.enqueued_at
214
+ end
215
+
216
+ it 'unwraps delayed jobs' do
217
+ Sidekiq::Queue.delay.foo(1,2,3)
218
+ q = Sidekiq::Queue.new
219
+ x = q.first
220
+ assert_equal "Sidekiq::Queue.foo", x.display_class
221
+ assert_equal [1,2,3], x.display_args
222
+ end
223
+
224
+ it 'unwraps ActiveJob jobs' do
225
+ ApiJob.perform_later(1, 2, 3)
226
+ q = Sidekiq::Queue.new
227
+ x = q.first
228
+ assert_equal "TestApi::ApiJob", x.display_class
229
+ assert_equal [1,2,3], x.display_args
230
+ end
231
+
232
+ it 'unwraps ActionMailer jobs' do
233
+ ApiMailer.test_email(1, 2, 3).deliver_later
234
+ q = Sidekiq::Queue.new('mailers')
235
+ x = q.first
236
+ assert_equal "TestApi::ApiMailer#test_email", x.display_class
237
+ assert_equal [1,2,3], x.display_args
238
+ end
239
+
240
+ it 'has no enqueued_at time for jobs enqueued in the future' do
241
+ job_id = ApiWorker.perform_in(100, 1, 'foo')
242
+ job = Sidekiq::ScheduledSet.new.find_job(job_id)
243
+ assert_nil job.enqueued_at
244
+ end
245
+
246
+ it 'can delete jobs' do
247
+ q = Sidekiq::Queue.new
248
+ ApiWorker.perform_async(1, 'mike')
249
+ assert_equal 1, q.size
250
+
251
+ x = q.first
252
+ assert_equal "TestApi::ApiWorker", x.display_class
253
+ assert_equal [1,'mike'], x.display_args
254
+
255
+ assert_equal [true], q.map(&:delete)
256
+ assert_equal 0, q.size
257
+ end
258
+
259
+ it "can move scheduled job to queue" do
260
+ remain_id = ApiWorker.perform_in(100, 1, 'jason')
261
+ job_id = ApiWorker.perform_in(100, 1, 'jason')
262
+ job = Sidekiq::ScheduledSet.new.find_job(job_id)
263
+ q = Sidekiq::Queue.new
264
+ job.add_to_queue
265
+ queued_job = q.find_job(job_id)
266
+ refute_nil queued_job
267
+ assert_equal queued_job.jid, job_id
268
+ assert_nil Sidekiq::ScheduledSet.new.find_job(job_id)
269
+ refute_nil Sidekiq::ScheduledSet.new.find_job(remain_id)
270
+ end
271
+
272
+ it "handles multiple scheduled jobs when moving to queue" do
273
+ jids = Sidekiq::Client.push_bulk('class' => ApiWorker,
274
+ 'args' => [[1, 'jason'], [2, 'jason']],
275
+ 'at' => Time.now.to_f)
276
+ assert_equal 2, jids.size
277
+ (remain_id, job_id) = jids
278
+ job = Sidekiq::ScheduledSet.new.find_job(job_id)
279
+ q = Sidekiq::Queue.new
280
+ job.add_to_queue
281
+ queued_job = q.find_job(job_id)
282
+ refute_nil queued_job
283
+ assert_equal queued_job.jid, job_id
284
+ assert_nil Sidekiq::ScheduledSet.new.find_job(job_id)
285
+ refute_nil Sidekiq::ScheduledSet.new.find_job(remain_id)
286
+ end
287
+
288
+ it 'can find job by id in sorted sets' do
289
+ job_id = ApiWorker.perform_in(100, 1, 'jason')
290
+ job = Sidekiq::ScheduledSet.new.find_job(job_id)
291
+ refute_nil job
292
+ assert_equal job_id, job.jid
293
+ assert_in_delta job.latency, 0.0, 0.1
294
+ end
295
+
296
+ it 'can remove jobs when iterating over a sorted set' do
297
+ # scheduled jobs must be greater than SortedSet#each underlying page size
298
+ 51.times do
299
+ ApiWorker.perform_in(100, 'aaron')
300
+ end
301
+ set = Sidekiq::ScheduledSet.new
302
+ set.map(&:delete)
303
+ assert_equal set.size, 0
304
+ end
305
+
306
+ it 'can remove jobs when iterating over a queue' do
307
+ # initial queue size must be greater than Queue#each underlying page size
308
+ 51.times do
309
+ ApiWorker.perform_async(1, 'aaron')
310
+ end
311
+ q = Sidekiq::Queue.new
312
+ q.map(&:delete)
313
+ assert_equal q.size, 0
314
+ end
315
+
316
+ it 'can find job by id in queues' do
317
+ q = Sidekiq::Queue.new
318
+ job_id = ApiWorker.perform_async(1, 'jason')
319
+ job = q.find_job(job_id)
320
+ refute_nil job
321
+ assert_equal job_id, job.jid
322
+ end
323
+
324
+ it 'can clear a queue' do
325
+ q = Sidekiq::Queue.new
326
+ 2.times { ApiWorker.perform_async(1, 'mike') }
327
+ q.clear
328
+
329
+ Sidekiq.redis do |conn|
330
+ refute conn.smembers('queues').include?('foo')
331
+ refute conn.exists('queue:foo')
332
+ end
333
+ end
334
+
335
+ it 'can fetch by score' do
336
+ same_time = Time.now.to_f
337
+ add_retry('bob1', same_time)
338
+ add_retry('bob2', same_time)
339
+ r = Sidekiq::RetrySet.new
340
+ assert_equal 2, r.fetch(same_time).size
341
+ end
342
+
343
+ it 'can fetch by score and jid' do
344
+ same_time = Time.now.to_f
345
+ add_retry('bob1', same_time)
346
+ add_retry('bob2', same_time)
347
+ r = Sidekiq::RetrySet.new
348
+ assert_equal 1, r.fetch(same_time, 'bob1').size
349
+ end
350
+
351
+ it 'shows empty retries' do
352
+ r = Sidekiq::RetrySet.new
353
+ assert_equal 0, r.size
354
+ end
355
+
356
+ it 'can enumerate retries' do
357
+ add_retry
358
+
359
+ r = Sidekiq::RetrySet.new
360
+ assert_equal 1, r.size
361
+ array = r.to_a
362
+ assert_equal 1, array.size
363
+
364
+ retri = array.first
365
+ assert_equal 'ApiWorker', retri.klass
366
+ assert_equal 'default', retri.queue
367
+ assert_equal 'bob', retri.jid
368
+ assert_in_delta Time.now.to_f, retri.at.to_f, 0.02
369
+ end
370
+
371
+ it 'requires a jid to delete an entry' do
372
+ start_time = Time.now.to_f
373
+ add_retry('bob2', Time.now.to_f)
374
+ assert_raises(ArgumentError) do
375
+ Sidekiq::RetrySet.new.delete(start_time)
376
+ end
377
+ end
378
+
379
+ it 'can delete a single retry from score and jid' do
380
+ same_time = Time.now.to_f
381
+ add_retry('bob1', same_time)
382
+ add_retry('bob2', same_time)
383
+ r = Sidekiq::RetrySet.new
384
+ assert_equal 2, r.size
385
+ Sidekiq::RetrySet.new.delete(same_time, 'bob1')
386
+ assert_equal 1, r.size
387
+ end
388
+
389
+ it 'can retry a retry' do
390
+ add_retry
391
+ r = Sidekiq::RetrySet.new
392
+ assert_equal 1, r.size
393
+ r.first.retry
394
+ assert_equal 0, r.size
395
+ assert_equal 1, Sidekiq::Queue.new('default').size
396
+ job = Sidekiq::Queue.new('default').first
397
+ assert_equal 'bob', job.jid
398
+ assert_equal 1, job['retry_count']
399
+ end
400
+
401
+ it 'can clear retries' do
402
+ add_retry
403
+ add_retry('test')
404
+ r = Sidekiq::RetrySet.new
405
+ assert_equal 2, r.size
406
+ r.clear
407
+ assert_equal 0, r.size
408
+ end
409
+
410
+ it 'can enumerate processes' do
411
+ identity_string = "identity_string"
412
+ odata = {
413
+ 'pid' => 123,
414
+ 'hostname' => Socket.gethostname,
415
+ 'key' => identity_string,
416
+ 'identity' => identity_string,
417
+ 'started_at' => Time.now.to_f - 15,
418
+ }
419
+
420
+ time = Time.now.to_f
421
+ Sidekiq.redis do |conn|
422
+ conn.multi do
423
+ conn.sadd('processes', odata['key'])
424
+ conn.hmset(odata['key'], 'info', Sidekiq.dump_json(odata), 'busy', 10, 'beat', time)
425
+ conn.sadd('processes', 'fake:pid')
426
+ end
427
+ end
428
+
429
+ ps = Sidekiq::ProcessSet.new.to_a
430
+ assert_equal 1, ps.size
431
+ data = ps.first
432
+ assert_equal 10, data['busy']
433
+ assert_equal time, data['beat']
434
+ assert_equal 123, data['pid']
435
+ data.quiet!
436
+ data.stop!
437
+ signals_string = "#{odata['key']}-signals"
438
+ assert_equal "TERM", Sidekiq.redis{|c| c.lpop(signals_string) }
439
+ assert_equal "USR1", Sidekiq.redis{|c| c.lpop(signals_string) }
440
+ end
441
+
442
+ it 'can enumerate workers' do
443
+ w = Sidekiq::Workers.new
444
+ assert_equal 0, w.size
445
+ w.each do
446
+ assert false
447
+ end
448
+
449
+ hn = Socket.gethostname
450
+ key = "#{hn}:#{$$}"
451
+ pdata = { 'pid' => $$, 'hostname' => hn, 'started_at' => Time.now.to_i }
452
+ Sidekiq.redis do |conn|
453
+ conn.sadd('processes', key)
454
+ conn.hmset(key, 'info', Sidekiq.dump_json(pdata), 'busy', 0, 'beat', Time.now.to_f)
455
+ end
456
+
457
+ s = "#{key}:workers"
458
+ data = Sidekiq.dump_json({ 'payload' => {}, 'queue' => 'default', 'run_at' => Time.now.to_i })
459
+ Sidekiq.redis do |c|
460
+ c.hmset(s, '1234', data)
461
+ end
462
+
463
+ w.each do |p, x, y|
464
+ assert_equal key, p
465
+ assert_equal "1234", x
466
+ assert_equal 'default', y['queue']
467
+ assert_equal Time.now.year, Time.at(y['run_at']).year
468
+ end
469
+
470
+ s = "#{key}:workers"
471
+ data = Sidekiq.dump_json({ 'payload' => {}, 'queue' => 'default', 'run_at' => (Time.now.to_i - 2*60*60) })
472
+ Sidekiq.redis do |c|
473
+ c.multi do
474
+ c.hmset(s, '5678', data)
475
+ c.hmset("b#{s}", '5678', data)
476
+ end
477
+ end
478
+
479
+ assert_equal ['1234', '5678'], w.map { |_, tid, _| tid }
480
+ end
481
+
482
+ it 'can reschedule jobs' do
483
+ add_retry('foo1')
484
+ add_retry('foo2')
485
+
486
+ retries = Sidekiq::RetrySet.new
487
+ assert_equal 2, retries.size
488
+ refute(retries.map { |r| r.score > (Time.now.to_f + 9) }.any?)
489
+
490
+ retries.each do |retri|
491
+ retri.reschedule(Time.now.to_f + 10) if retri.jid == 'foo2'
492
+ end
493
+
494
+ assert_equal 2, retries.size
495
+ assert(retries.map { |r| r.score > (Time.now.to_f + 9) }.any?)
496
+ end
497
+
498
+ it 'prunes processes which have died' do
499
+ data = { 'pid' => rand(10_000), 'hostname' => "app#{rand(1_000)}", 'started_at' => Time.now.to_f }
500
+ key = "#{data['hostname']}:#{data['pid']}"
501
+ Sidekiq.redis do |conn|
502
+ conn.sadd('processes', key)
503
+ conn.hmset(key, 'info', Sidekiq.dump_json(data), 'busy', 0, 'beat', Time.now.to_f)
504
+ end
505
+
506
+ ps = Sidekiq::ProcessSet.new
507
+ assert_equal 1, ps.size
508
+ assert_equal 1, ps.to_a.size
509
+
510
+ Sidekiq.redis do |conn|
511
+ conn.sadd('processes', "bar:987")
512
+ conn.sadd('processes', "bar:986")
513
+ end
514
+
515
+ ps = Sidekiq::ProcessSet.new
516
+ assert_equal 1, ps.size
517
+ assert_equal 1, ps.to_a.size
518
+ end
519
+
520
+ def add_retry(jid = 'bob', at = Time.now.to_f)
521
+ payload = Sidekiq.dump_json('class' => 'ApiWorker', 'args' => [1, 'mike'], 'queue' => 'default', 'jid' => jid, 'retry_count' => 2, 'failed_at' => Time.now.to_f)
522
+ Sidekiq.redis do |conn|
523
+ conn.zadd('retry', at.to_s, payload)
524
+ end
525
+ end
526
+ end
527
+ end
528
+ end