sr-sidekiq 4.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +12 -0
  3. data/3.0-Upgrade.md +70 -0
  4. data/4.0-Upgrade.md +50 -0
  5. data/COMM-LICENSE (sidekiq) +95 -0
  6. data/Changes.md +1241 -0
  7. data/Ent-Changes.md +112 -0
  8. data/Gemfile +29 -0
  9. data/LICENSE (sidekiq) +9 -0
  10. data/LICENSE (sr-sidekiq) +5 -0
  11. data/Pro-2.0-Upgrade.md +138 -0
  12. data/Pro-3.0-Upgrade.md +44 -0
  13. data/Pro-Changes.md +539 -0
  14. data/README.md +8 -0
  15. data/Rakefile +9 -0
  16. data/bin/sidekiq +18 -0
  17. data/bin/sidekiqctl +99 -0
  18. data/bin/sidekiqload +167 -0
  19. data/code_of_conduct.md +50 -0
  20. data/lib/generators/sidekiq/templates/worker.rb.erb +9 -0
  21. data/lib/generators/sidekiq/templates/worker_spec.rb.erb +6 -0
  22. data/lib/generators/sidekiq/templates/worker_test.rb.erb +8 -0
  23. data/lib/generators/sidekiq/worker_generator.rb +49 -0
  24. data/lib/sidekiq.rb +237 -0
  25. data/lib/sidekiq/api.rb +844 -0
  26. data/lib/sidekiq/cli.rb +389 -0
  27. data/lib/sidekiq/client.rb +260 -0
  28. data/lib/sidekiq/core_ext.rb +106 -0
  29. data/lib/sidekiq/exception_handler.rb +31 -0
  30. data/lib/sidekiq/extensions/action_mailer.rb +57 -0
  31. data/lib/sidekiq/extensions/active_record.rb +40 -0
  32. data/lib/sidekiq/extensions/class_methods.rb +40 -0
  33. data/lib/sidekiq/extensions/generic_proxy.rb +25 -0
  34. data/lib/sidekiq/fetch.rb +81 -0
  35. data/lib/sidekiq/launcher.rb +160 -0
  36. data/lib/sidekiq/logging.rb +106 -0
  37. data/lib/sidekiq/manager.rb +137 -0
  38. data/lib/sidekiq/middleware/chain.rb +150 -0
  39. data/lib/sidekiq/middleware/i18n.rb +42 -0
  40. data/lib/sidekiq/middleware/server/active_record.rb +13 -0
  41. data/lib/sidekiq/middleware/server/logging.rb +40 -0
  42. data/lib/sidekiq/middleware/server/retry_jobs.rb +205 -0
  43. data/lib/sidekiq/paginator.rb +43 -0
  44. data/lib/sidekiq/processor.rb +186 -0
  45. data/lib/sidekiq/rails.rb +39 -0
  46. data/lib/sidekiq/redis_connection.rb +97 -0
  47. data/lib/sidekiq/scheduled.rb +146 -0
  48. data/lib/sidekiq/testing.rb +316 -0
  49. data/lib/sidekiq/testing/inline.rb +29 -0
  50. data/lib/sidekiq/util.rb +62 -0
  51. data/lib/sidekiq/version.rb +4 -0
  52. data/lib/sidekiq/web.rb +278 -0
  53. data/lib/sidekiq/web_helpers.rb +255 -0
  54. data/lib/sidekiq/worker.rb +121 -0
  55. data/sidekiq.gemspec +26 -0
  56. data/sr-sidekiq-4.1.3.gem +0 -0
  57. data/sr-sidekiq-4.1.4.gem +0 -0
  58. data/sr-sidekiq-4.1.5.gem +0 -0
  59. data/test/config.yml +9 -0
  60. data/test/env_based_config.yml +11 -0
  61. data/test/fake_env.rb +1 -0
  62. data/test/fixtures/en.yml +2 -0
  63. data/test/helper.rb +75 -0
  64. data/test/test_actors.rb +138 -0
  65. data/test/test_api.rb +528 -0
  66. data/test/test_cli.rb +406 -0
  67. data/test/test_client.rb +262 -0
  68. data/test/test_exception_handler.rb +56 -0
  69. data/test/test_extensions.rb +127 -0
  70. data/test/test_fetch.rb +50 -0
  71. data/test/test_launcher.rb +85 -0
  72. data/test/test_logging.rb +35 -0
  73. data/test/test_manager.rb +50 -0
  74. data/test/test_middleware.rb +158 -0
  75. data/test/test_processor.rb +201 -0
  76. data/test/test_rails.rb +22 -0
  77. data/test/test_redis_connection.rb +127 -0
  78. data/test/test_retry.rb +326 -0
  79. data/test/test_retry_exhausted.rb +149 -0
  80. data/test/test_scheduled.rb +115 -0
  81. data/test/test_scheduling.rb +50 -0
  82. data/test/test_sidekiq.rb +107 -0
  83. data/test/test_testing.rb +143 -0
  84. data/test/test_testing_fake.rb +357 -0
  85. data/test/test_testing_inline.rb +94 -0
  86. data/test/test_util.rb +13 -0
  87. data/test/test_web.rb +614 -0
  88. data/test/test_web_helpers.rb +54 -0
  89. data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
  90. data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
  91. data/web/assets/images/favicon.ico +0 -0
  92. data/web/assets/images/logo.png +0 -0
  93. data/web/assets/images/status-sd8051fd480.png +0 -0
  94. data/web/assets/images/status/active.png +0 -0
  95. data/web/assets/images/status/idle.png +0 -0
  96. data/web/assets/javascripts/application.js +88 -0
  97. data/web/assets/javascripts/dashboard.js +300 -0
  98. data/web/assets/javascripts/locales/README.md +27 -0
  99. data/web/assets/javascripts/locales/jquery.timeago.ar.js +96 -0
  100. data/web/assets/javascripts/locales/jquery.timeago.bg.js +18 -0
  101. data/web/assets/javascripts/locales/jquery.timeago.bs.js +49 -0
  102. data/web/assets/javascripts/locales/jquery.timeago.ca.js +18 -0
  103. data/web/assets/javascripts/locales/jquery.timeago.cs.js +18 -0
  104. data/web/assets/javascripts/locales/jquery.timeago.cy.js +20 -0
  105. data/web/assets/javascripts/locales/jquery.timeago.da.js +18 -0
  106. data/web/assets/javascripts/locales/jquery.timeago.de.js +18 -0
  107. data/web/assets/javascripts/locales/jquery.timeago.el.js +18 -0
  108. data/web/assets/javascripts/locales/jquery.timeago.en-short.js +20 -0
  109. data/web/assets/javascripts/locales/jquery.timeago.en.js +20 -0
  110. data/web/assets/javascripts/locales/jquery.timeago.es.js +18 -0
  111. data/web/assets/javascripts/locales/jquery.timeago.et.js +18 -0
  112. data/web/assets/javascripts/locales/jquery.timeago.fa.js +22 -0
  113. data/web/assets/javascripts/locales/jquery.timeago.fi.js +28 -0
  114. data/web/assets/javascripts/locales/jquery.timeago.fr-short.js +16 -0
  115. data/web/assets/javascripts/locales/jquery.timeago.fr.js +17 -0
  116. data/web/assets/javascripts/locales/jquery.timeago.he.js +18 -0
  117. data/web/assets/javascripts/locales/jquery.timeago.hr.js +49 -0
  118. data/web/assets/javascripts/locales/jquery.timeago.hu.js +18 -0
  119. data/web/assets/javascripts/locales/jquery.timeago.hy.js +18 -0
  120. data/web/assets/javascripts/locales/jquery.timeago.id.js +18 -0
  121. data/web/assets/javascripts/locales/jquery.timeago.it.js +16 -0
  122. data/web/assets/javascripts/locales/jquery.timeago.ja.js +19 -0
  123. data/web/assets/javascripts/locales/jquery.timeago.ko.js +17 -0
  124. data/web/assets/javascripts/locales/jquery.timeago.lt.js +20 -0
  125. data/web/assets/javascripts/locales/jquery.timeago.mk.js +20 -0
  126. data/web/assets/javascripts/locales/jquery.timeago.nl.js +20 -0
  127. data/web/assets/javascripts/locales/jquery.timeago.no.js +18 -0
  128. data/web/assets/javascripts/locales/jquery.timeago.pl.js +31 -0
  129. data/web/assets/javascripts/locales/jquery.timeago.pt-br.js +16 -0
  130. data/web/assets/javascripts/locales/jquery.timeago.pt.js +16 -0
  131. data/web/assets/javascripts/locales/jquery.timeago.ro.js +18 -0
  132. data/web/assets/javascripts/locales/jquery.timeago.rs.js +49 -0
  133. data/web/assets/javascripts/locales/jquery.timeago.ru.js +34 -0
  134. data/web/assets/javascripts/locales/jquery.timeago.sk.js +18 -0
  135. data/web/assets/javascripts/locales/jquery.timeago.sl.js +44 -0
  136. data/web/assets/javascripts/locales/jquery.timeago.sv.js +18 -0
  137. data/web/assets/javascripts/locales/jquery.timeago.th.js +20 -0
  138. data/web/assets/javascripts/locales/jquery.timeago.tr.js +16 -0
  139. data/web/assets/javascripts/locales/jquery.timeago.uk.js +34 -0
  140. data/web/assets/javascripts/locales/jquery.timeago.uz.js +19 -0
  141. data/web/assets/javascripts/locales/jquery.timeago.zh-cn.js +20 -0
  142. data/web/assets/javascripts/locales/jquery.timeago.zh-tw.js +20 -0
  143. data/web/assets/stylesheets/application.css +754 -0
  144. data/web/assets/stylesheets/bootstrap.css +9 -0
  145. data/web/locales/cs.yml +78 -0
  146. data/web/locales/da.yml +68 -0
  147. data/web/locales/de.yml +69 -0
  148. data/web/locales/el.yml +68 -0
  149. data/web/locales/en.yml +79 -0
  150. data/web/locales/es.yml +69 -0
  151. data/web/locales/fr.yml +78 -0
  152. data/web/locales/hi.yml +75 -0
  153. data/web/locales/it.yml +69 -0
  154. data/web/locales/ja.yml +78 -0
  155. data/web/locales/ko.yml +68 -0
  156. data/web/locales/nb.yml +77 -0
  157. data/web/locales/nl.yml +68 -0
  158. data/web/locales/pl.yml +59 -0
  159. data/web/locales/pt-br.yml +68 -0
  160. data/web/locales/pt.yml +67 -0
  161. data/web/locales/ru.yml +78 -0
  162. data/web/locales/sv.yml +68 -0
  163. data/web/locales/ta.yml +75 -0
  164. data/web/locales/uk.yml +76 -0
  165. data/web/locales/zh-cn.yml +68 -0
  166. data/web/locales/zh-tw.yml +68 -0
  167. data/web/views/_footer.erb +17 -0
  168. data/web/views/_job_info.erb +88 -0
  169. data/web/views/_nav.erb +66 -0
  170. data/web/views/_paging.erb +23 -0
  171. data/web/views/_poll_js.erb +5 -0
  172. data/web/views/_poll_link.erb +7 -0
  173. data/web/views/_status.erb +4 -0
  174. data/web/views/_summary.erb +40 -0
  175. data/web/views/busy.erb +94 -0
  176. data/web/views/dashboard.erb +75 -0
  177. data/web/views/dead.erb +34 -0
  178. data/web/views/layout.erb +32 -0
  179. data/web/views/morgue.erb +71 -0
  180. data/web/views/queue.erb +45 -0
  181. data/web/views/queues.erb +28 -0
  182. data/web/views/retries.erb +74 -0
  183. data/web/views/retry.erb +34 -0
  184. data/web/views/scheduled.erb +54 -0
  185. data/web/views/scheduled_job_info.erb +8 -0
  186. metadata +408 -0
@@ -0,0 +1,121 @@
1
+ # frozen_string_literal: true
2
+ require 'sidekiq/client'
3
+ require 'sidekiq/core_ext'
4
+
5
+ module Sidekiq
6
+
7
+ ##
8
+ # Include this module in your worker class and you can easily create
9
+ # asynchronous jobs:
10
+ #
11
+ # class HardWorker
12
+ # include Sidekiq::Worker
13
+ #
14
+ # def perform(*args)
15
+ # # do some work
16
+ # end
17
+ # end
18
+ #
19
+ # Then in your Rails app, you can do this:
20
+ #
21
+ # HardWorker.perform_async(1, 2, 3)
22
+ #
23
+ # Note that perform_async is a class method, perform is an instance method.
24
+ module Worker
25
+ attr_accessor :jid
26
+
27
+ def self.included(base)
28
+ raise ArgumentError, "You cannot include Sidekiq::Worker in an ActiveJob: #{base.name}" if base.ancestors.any? {|c| c.name == 'ActiveJob::Base' }
29
+
30
+ base.extend(ClassMethods)
31
+ base.class_attribute :sidekiq_options_hash
32
+ base.class_attribute :sidekiq_retry_in_block
33
+ base.class_attribute :sidekiq_retries_exhausted_block
34
+ end
35
+
36
+ def logger
37
+ Sidekiq.logger
38
+ end
39
+
40
+ module ClassMethods
41
+
42
+ def delay(*args)
43
+ raise ArgumentError, "Do not call .delay on a Sidekiq::Worker class, call .perform_async"
44
+ end
45
+
46
+ def delay_for(*args)
47
+ raise ArgumentError, "Do not call .delay_for on a Sidekiq::Worker class, call .perform_in"
48
+ end
49
+
50
+ def delay_until(*args)
51
+ raise ArgumentError, "Do not call .delay_until on a Sidekiq::Worker class, call .perform_at"
52
+ end
53
+
54
+ def set(options)
55
+ Thread.current[:sidekiq_worker_set] = options
56
+ self
57
+ end
58
+
59
+ def perform_async(*args)
60
+ client_push('class' => self, 'args' => args)
61
+ end
62
+
63
+ # +interval+ must be a timestamp, numeric or something that acts
64
+ # numeric (like an activesupport time interval).
65
+ def perform_in(interval, *args)
66
+ int = interval.to_f
67
+ now = Time.now
68
+ ts = (int < 1_000_000_000 ? (now + interval).to_f : int)
69
+
70
+ item = { 'class' => self, 'args' => args, 'at' => ts }
71
+
72
+ # Optimization to enqueue something now that is scheduled to go out now or in the past
73
+ item.delete('at'.freeze) if ts <= now.to_f
74
+
75
+ client_push(item)
76
+ end
77
+ alias_method :perform_at, :perform_in
78
+
79
+ ##
80
+ # Allows customization for this type of Worker.
81
+ # Legal options:
82
+ #
83
+ # queue - use a named queue for this Worker, default 'default'
84
+ # retry - enable the RetryJobs middleware for this Worker, *true* to use the default
85
+ # or *Integer* count
86
+ # backtrace - whether to save any error backtrace in the retry payload to display in web UI,
87
+ # can be true, false or an integer number of lines to save, default *false*
88
+ # pool - use the given Redis connection pool to push this type of job to a given shard.
89
+ #
90
+ # In practice, any option is allowed. This is the main mechanism to configure the
91
+ # options for a specific job.
92
+ def sidekiq_options(opts={})
93
+ self.sidekiq_options_hash = get_sidekiq_options.merge(opts.stringify_keys)
94
+ end
95
+
96
+ def sidekiq_retry_in(&block)
97
+ self.sidekiq_retry_in_block = block
98
+ end
99
+
100
+ def sidekiq_retries_exhausted(&block)
101
+ self.sidekiq_retries_exhausted_block = block
102
+ end
103
+
104
+ def get_sidekiq_options # :nodoc:
105
+ self.sidekiq_options_hash ||= Sidekiq.default_worker_options
106
+ end
107
+
108
+ def client_push(item) # :nodoc:
109
+ pool = Thread.current[:sidekiq_via_pool] || get_sidekiq_options['pool'] || Sidekiq.redis_pool
110
+ hash = if Thread.current[:sidekiq_worker_set]
111
+ x, Thread.current[:sidekiq_worker_set] = Thread.current[:sidekiq_worker_set], nil
112
+ x.stringify_keys.merge(item.stringify_keys)
113
+ else
114
+ item.stringify_keys
115
+ end
116
+ Sidekiq::Client.new(pool).push(hash)
117
+ end
118
+
119
+ end
120
+ end
121
+ end
data/sidekiq.gemspec ADDED
@@ -0,0 +1,26 @@
1
+ # -*- encoding: utf-8 -*-
2
+ require File.expand_path('../lib/sidekiq/version', __FILE__)
3
+
4
+ Gem::Specification.new do |gem|
5
+ gem.authors = ["Sean Huber"]
6
+ gem.email = ["seanhuber@seanhuber.com"]
7
+ gem.summary = "A fork of Sidekiq (v4.1.2) with worker parameter restrictions"
8
+ gem.description = "A fork of Sidekiq (v4.1.2) with worker parameter restrictions"
9
+ gem.homepage = "https://github.com/seanhuber/sr-sidekiq"
10
+ gem.license = "LGPL-3.0"
11
+
12
+ gem.executables = ['sidekiq', 'sidekiqctl']
13
+ gem.files = `git ls-files | grep -Ev '^(myapp|examples)'`.split("\n")
14
+ gem.test_files = `git ls-files -- test/*`.split("\n")
15
+ gem.name = "sr-sidekiq"
16
+ gem.require_paths = ["lib"]
17
+ gem.version = Sidekiq::VERSION
18
+ gem.add_dependency 'redis', '~> 3.2', '>= 3.2.1'
19
+ gem.add_dependency 'connection_pool', '~> 2.2', '>= 2.2.0'
20
+ gem.add_dependency 'concurrent-ruby', '~> 1.0'
21
+ gem.add_development_dependency 'redis-namespace', '~> 1.5', '>= 1.5.2'
22
+ gem.add_development_dependency 'sinatra', '~> 1.4', '>= 1.4.6'
23
+ gem.add_development_dependency 'minitest', '~> 5.7', '>= 5.7.0'
24
+ gem.add_development_dependency 'rake', '~> 10.0'
25
+ gem.add_development_dependency 'rails', '~> 4', '>= 3.2.0'
26
+ end
Binary file
Binary file
Binary file
data/test/config.yml ADDED
@@ -0,0 +1,9 @@
1
+ ---
2
+ :verbose: false
3
+ :require: ./test/fake_env.rb
4
+ :pidfile: /tmp/sidekiq-config-test.pid
5
+ :logfile: /tmp/sidekiq.log
6
+ :concurrency: 50
7
+ :queues:
8
+ - [<%="very_"%>often, 2]
9
+ - [seldom, 1]
@@ -0,0 +1,11 @@
1
+ ---
2
+ :pidfile: /tmp/sidekiq-config-test.pid
3
+ :concurrency: 50
4
+ staging:
5
+ :verbose: false
6
+ :require: ./test/fake_env.rb
7
+ :logfile: /tmp/sidekiq.log
8
+ :concurrency: 5
9
+ :queues:
10
+ - [<%="very_"%>often, 2]
11
+ - [seldom, 1]
data/test/fake_env.rb ADDED
@@ -0,0 +1 @@
1
+ # frozen_string_literal: true
@@ -0,0 +1,2 @@
1
+ en:
2
+ translated_text: 'Changed text from add locals'
data/test/helper.rb ADDED
@@ -0,0 +1,75 @@
1
+ # frozen_string_literal: true
2
+ $TESTING = true
3
+ # disable minitest/parallel threads
4
+ ENV["N"] = "0"
5
+
6
+ if ENV["COVERAGE"]
7
+ require 'simplecov'
8
+ SimpleCov.start do
9
+ add_filter "/test/"
10
+ add_filter "/myapp/"
11
+ end
12
+ end
13
+ ENV['RACK_ENV'] = ENV['RAILS_ENV'] = 'test'
14
+
15
+ trap 'USR1' do
16
+ threads = Thread.list
17
+
18
+ puts
19
+ puts "=" * 80
20
+ puts "Received USR1 signal; printing all #{threads.count} thread backtraces."
21
+
22
+ threads.each do |thr|
23
+ description = thr == Thread.main ? "Main thread" : thr.inspect
24
+ puts
25
+ puts "#{description} backtrace: "
26
+ puts thr.backtrace.join("\n")
27
+ end
28
+
29
+ puts "=" * 80
30
+ end
31
+
32
+ begin
33
+ require 'pry-byebug'
34
+ rescue LoadError
35
+ end
36
+
37
+ require 'minitest/autorun'
38
+
39
+ require 'sidekiq'
40
+ require 'sidekiq/util'
41
+ Sidekiq.logger.level = Logger::ERROR
42
+
43
+ Sidekiq::Test = Minitest::Test
44
+
45
+ require 'sidekiq/redis_connection'
46
+ REDIS_URL = ENV['REDIS_URL'] || 'redis://localhost/15'
47
+ REDIS = Sidekiq::RedisConnection.create(:url => REDIS_URL, :namespace => 'testy')
48
+
49
+ Sidekiq.configure_client do |config|
50
+ config.redis = { :url => REDIS_URL, :namespace => 'testy' }
51
+ end
52
+
53
+ def capture_logging(lvl=Logger::INFO)
54
+ old = Sidekiq.logger
55
+ begin
56
+ out = StringIO.new
57
+ logger = Logger.new(out)
58
+ logger.level = lvl
59
+ Sidekiq.logger = logger
60
+ yield
61
+ out.string
62
+ ensure
63
+ Sidekiq.logger = old
64
+ end
65
+ end
66
+
67
+ def with_logging(lvl=Logger::DEBUG)
68
+ old = Sidekiq.logger.level
69
+ begin
70
+ Sidekiq.logger.level = lvl
71
+ yield
72
+ ensure
73
+ Sidekiq.logger.level = old
74
+ end
75
+ end
@@ -0,0 +1,138 @@
1
+ # frozen_string_literal: true
2
+ require_relative 'helper'
3
+ require 'sidekiq/cli'
4
+ require 'sidekiq/fetch'
5
+ require 'sidekiq/scheduled'
6
+ require 'sidekiq/processor'
7
+
8
+ class TestActors < Sidekiq::Test
9
+ class JoeWorker
10
+ include Sidekiq::Worker
11
+ def perform(slp)
12
+ raise "boom" if slp == "boom"
13
+ sleep(slp) if slp > 0
14
+ $count += 1
15
+ end
16
+ end
17
+
18
+ describe 'threads' do
19
+ before do
20
+ Sidekiq.redis {|c| c.flushdb}
21
+ end
22
+
23
+ describe 'scheduler' do
24
+ it 'can start and stop' do
25
+ f = Sidekiq::Scheduled::Poller.new
26
+ f.start
27
+ f.terminate
28
+ end
29
+
30
+ it 'can schedule' do
31
+ ss = Sidekiq::ScheduledSet.new
32
+ q = Sidekiq::Queue.new
33
+
34
+ JoeWorker.perform_in(0.01, 0)
35
+
36
+ assert_equal 0, q.size
37
+ assert_equal 1, ss.size
38
+
39
+ sleep 0.015
40
+ s = Sidekiq::Scheduled::Poller.new
41
+ s.enqueue
42
+ assert_equal 1, q.size
43
+ assert_equal 0, ss.size
44
+ s.terminate
45
+ end
46
+ end
47
+
48
+ describe 'processor' do
49
+ before do
50
+ $count = 0
51
+ end
52
+
53
+ it 'can start and stop' do
54
+ f = Sidekiq::Processor.new(Mgr.new)
55
+ f.terminate
56
+ end
57
+
58
+ class Mgr
59
+ attr_reader :latest_error
60
+ attr_reader :mutex
61
+ attr_reader :cond
62
+ def initialize
63
+ @mutex = ::Mutex.new
64
+ @cond = ::ConditionVariable.new
65
+ end
66
+ def processor_died(inst, err)
67
+ @latest_error = err
68
+ @mutex.synchronize do
69
+ @cond.signal
70
+ end
71
+ end
72
+ def processor_stopped(inst)
73
+ @mutex.synchronize do
74
+ @cond.signal
75
+ end
76
+ end
77
+ def options
78
+ { :concurrency => 3, :queues => ['default'] }
79
+ end
80
+ end
81
+
82
+ it 'can process' do
83
+ mgr = Mgr.new
84
+
85
+ p = Sidekiq::Processor.new(mgr)
86
+ JoeWorker.perform_async(0)
87
+
88
+ a = $count
89
+ p.process_one
90
+ b = $count
91
+ assert_equal a + 1, b
92
+ end
93
+
94
+ it 'deals with errors' do
95
+ mgr = Mgr.new
96
+
97
+ p = Sidekiq::Processor.new(mgr)
98
+ JoeWorker.perform_async("boom")
99
+ q = Sidekiq::Queue.new
100
+ assert_equal 1, q.size
101
+
102
+ a = $count
103
+ mgr.mutex.synchronize do
104
+ p.start
105
+ mgr.cond.wait(mgr.mutex)
106
+ end
107
+ b = $count
108
+ assert_equal a, b
109
+
110
+ sleep 0.001
111
+ assert_equal false, p.thread.status
112
+ p.terminate(true)
113
+ refute_nil mgr.latest_error
114
+ assert_equal RuntimeError, mgr.latest_error.class
115
+ end
116
+
117
+ it 'gracefully kills' do
118
+ mgr = Mgr.new
119
+
120
+ p = Sidekiq::Processor.new(mgr)
121
+ JoeWorker.perform_async(1)
122
+ q = Sidekiq::Queue.new
123
+ assert_equal 1, q.size
124
+
125
+ a = $count
126
+ p.start
127
+ sleep(0.02)
128
+ p.terminate
129
+ p.kill(true)
130
+
131
+ b = $count
132
+ assert_equal a, b
133
+ assert_equal false, p.thread.status
134
+ refute mgr.latest_error, mgr.latest_error.to_s
135
+ end
136
+ end
137
+ end
138
+ end
data/test/test_api.rb ADDED
@@ -0,0 +1,528 @@
1
+ # frozen_string_literal: true
2
+ require_relative 'helper'
3
+ require 'sidekiq/api'
4
+ require 'active_job'
5
+ require 'action_mailer'
6
+
7
+ class TestApi < Sidekiq::Test
8
+ describe 'api' do
9
+ before do
10
+ Sidekiq.redis {|c| c.flushdb }
11
+ end
12
+
13
+ describe "stats" do
14
+ it "is initially zero" do
15
+ s = Sidekiq::Stats.new
16
+ assert_equal 0, s.processed
17
+ assert_equal 0, s.failed
18
+ assert_equal 0, s.enqueued
19
+ end
20
+
21
+ describe "processed" do
22
+ it "returns number of processed jobs" do
23
+ Sidekiq.redis { |conn| conn.set("stat:processed", 5) }
24
+ s = Sidekiq::Stats.new
25
+ assert_equal 5, s.processed
26
+ end
27
+ end
28
+
29
+ describe "failed" do
30
+ it "returns number of failed jobs" do
31
+ Sidekiq.redis { |conn| conn.set("stat:failed", 5) }
32
+ s = Sidekiq::Stats.new
33
+ assert_equal 5, s.failed
34
+ end
35
+ end
36
+
37
+ describe "reset" do
38
+ before do
39
+ Sidekiq.redis do |conn|
40
+ conn.set('stat:processed', 5)
41
+ conn.set('stat:failed', 10)
42
+ end
43
+ end
44
+
45
+ it 'will reset all stats by default' do
46
+ Sidekiq::Stats.new.reset
47
+ s = Sidekiq::Stats.new
48
+ assert_equal 0, s.failed
49
+ assert_equal 0, s.processed
50
+ end
51
+
52
+ it 'can reset individual stats' do
53
+ Sidekiq::Stats.new.reset('failed')
54
+ s = Sidekiq::Stats.new
55
+ assert_equal 0, s.failed
56
+ assert_equal 5, s.processed
57
+ end
58
+
59
+ it 'can accept anything that responds to #to_s' do
60
+ Sidekiq::Stats.new.reset(:failed)
61
+ s = Sidekiq::Stats.new
62
+ assert_equal 0, s.failed
63
+ assert_equal 5, s.processed
64
+ end
65
+
66
+ it 'ignores anything other than "failed" or "processed"' do
67
+ Sidekiq::Stats.new.reset((1..10).to_a, ['failed'])
68
+ s = Sidekiq::Stats.new
69
+ assert_equal 0, s.failed
70
+ assert_equal 5, s.processed
71
+ end
72
+ end
73
+
74
+ describe "queues" do
75
+ it "is initially empty" do
76
+ s = Sidekiq::Stats::Queues.new
77
+ assert_equal 0, s.lengths.size
78
+ end
79
+
80
+ it "returns a hash of queue and size in order" do
81
+ Sidekiq.redis do |conn|
82
+ conn.rpush 'queue:foo', '{}'
83
+ conn.sadd 'queues', 'foo'
84
+
85
+ 3.times { conn.rpush 'queue:bar', '{}' }
86
+ conn.sadd 'queues', 'bar'
87
+ end
88
+
89
+ s = Sidekiq::Stats::Queues.new
90
+ assert_equal ({ "foo" => 1, "bar" => 3 }), s.lengths
91
+ assert_equal "bar", s.lengths.first.first
92
+
93
+ assert_equal Sidekiq::Stats.new.queues, Sidekiq::Stats::Queues.new.lengths
94
+ end
95
+ end
96
+
97
+ describe "enqueued" do
98
+ it "returns total enqueued jobs" do
99
+ Sidekiq.redis do |conn|
100
+ conn.rpush 'queue:foo', '{}'
101
+ conn.sadd 'queues', 'foo'
102
+
103
+ 3.times { conn.rpush 'queue:bar', '{}' }
104
+ conn.sadd 'queues', 'bar'
105
+ end
106
+
107
+ s = Sidekiq::Stats.new
108
+ assert_equal 4, s.enqueued
109
+ end
110
+ end
111
+
112
+ describe "over time" do
113
+ before do
114
+ require 'active_support/core_ext/time/conversions'
115
+ @before = Time::DATE_FORMATS[:default]
116
+ Time::DATE_FORMATS[:default] = "%d/%m/%Y %H:%M:%S"
117
+ end
118
+
119
+ after do
120
+ Time::DATE_FORMATS[:default] = @before
121
+ end
122
+
123
+ describe "processed" do
124
+ it 'retrieves hash of dates' do
125
+ Sidekiq.redis do |c|
126
+ c.incrby("stat:processed:2012-12-24", 4)
127
+ c.incrby("stat:processed:2012-12-25", 1)
128
+ c.incrby("stat:processed:2012-12-26", 6)
129
+ c.incrby("stat:processed:2012-12-27", 2)
130
+ end
131
+ Time.stub(:now, Time.parse("2012-12-26 1:00:00 -0500")) do
132
+ s = Sidekiq::Stats::History.new(2)
133
+ assert_equal({ "2012-12-26" => 6, "2012-12-25" => 1 }, s.processed)
134
+
135
+ s = Sidekiq::Stats::History.new(3)
136
+ assert_equal({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }, s.processed)
137
+
138
+ s = Sidekiq::Stats::History.new(2, Date.parse("2012-12-25"))
139
+ assert_equal({ "2012-12-25" => 1, "2012-12-24" => 4 }, s.processed)
140
+ end
141
+ end
142
+ end
143
+
144
+ describe "failed" do
145
+ it 'retrieves hash of dates' do
146
+ Sidekiq.redis do |c|
147
+ c.incrby("stat:failed:2012-12-24", 4)
148
+ c.incrby("stat:failed:2012-12-25", 1)
149
+ c.incrby("stat:failed:2012-12-26", 6)
150
+ c.incrby("stat:failed:2012-12-27", 2)
151
+ end
152
+ Time.stub(:now, Time.parse("2012-12-26 1:00:00 -0500")) do
153
+ s = Sidekiq::Stats::History.new(2)
154
+ assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1 }), s.failed
155
+
156
+ s = Sidekiq::Stats::History.new(3)
157
+ assert_equal ({ "2012-12-26" => 6, "2012-12-25" => 1, "2012-12-24" => 4 }), s.failed
158
+
159
+ s = Sidekiq::Stats::History.new(2, Date.parse("2012-12-25"))
160
+ assert_equal ({ "2012-12-25" => 1, "2012-12-24" => 4 }), s.failed
161
+ end
162
+ end
163
+ end
164
+ end
165
+ end
166
+
167
+ describe 'with an empty database' do
168
+ it 'shows queue as empty' do
169
+ q = Sidekiq::Queue.new
170
+ assert_equal 0, q.size
171
+ assert_equal 0, q.latency
172
+ end
173
+
174
+ before do
175
+ ActiveJob::Base.queue_adapter = :sidekiq
176
+ ActiveJob::Base.logger = nil
177
+
178
+ class ApiMailer < ActionMailer::Base
179
+ def test_email(*)
180
+ end
181
+ end
182
+
183
+ class ApiJob < ActiveJob::Base
184
+ def perform(*)
185
+ end
186
+ end
187
+ end
188
+
189
+ class ApiWorker
190
+ include Sidekiq::Worker
191
+ end
192
+
193
+ it 'can enumerate jobs' do
194
+ q = Sidekiq::Queue.new
195
+ Time.stub(:now, Time.new(2012, 12, 26)) do
196
+ ApiWorker.perform_async(1, 'mike')
197
+ assert_equal ['TestApi::ApiWorker'], q.map(&:klass)
198
+
199
+ job = q.first
200
+ assert_equal 24, job.jid.size
201
+ assert_equal [1, 'mike'], job.args
202
+ assert_equal Time.new(2012, 12, 26), job.enqueued_at
203
+ end
204
+ assert q.latency > 10_000_000
205
+
206
+ q = Sidekiq::Queue.new('other')
207
+ assert_equal 0, q.size
208
+ end
209
+
210
+ it 'has no enqueued_at time for jobs enqueued in the future' do
211
+ job_id = ApiWorker.perform_in(100, 1, 'foo')
212
+ job = Sidekiq::ScheduledSet.new.find_job(job_id)
213
+ assert_nil job.enqueued_at
214
+ end
215
+
216
+ it 'unwraps delayed jobs' do
217
+ Sidekiq::Queue.delay.foo(1,2,3)
218
+ q = Sidekiq::Queue.new
219
+ x = q.first
220
+ assert_equal "Sidekiq::Queue.foo", x.display_class
221
+ assert_equal [1,2,3], x.display_args
222
+ end
223
+
224
+ it 'unwraps ActiveJob jobs' do
225
+ ApiJob.perform_later(1, 2, 3)
226
+ q = Sidekiq::Queue.new
227
+ x = q.first
228
+ assert_equal "TestApi::ApiJob", x.display_class
229
+ assert_equal [1,2,3], x.display_args
230
+ end
231
+
232
+ it 'unwraps ActionMailer jobs' do
233
+ ApiMailer.test_email(1, 2, 3).deliver_later
234
+ q = Sidekiq::Queue.new('mailers')
235
+ x = q.first
236
+ assert_equal "TestApi::ApiMailer#test_email", x.display_class
237
+ assert_equal [1,2,3], x.display_args
238
+ end
239
+
240
+ it 'has no enqueued_at time for jobs enqueued in the future' do
241
+ job_id = ApiWorker.perform_in(100, 1, 'foo')
242
+ job = Sidekiq::ScheduledSet.new.find_job(job_id)
243
+ assert_nil job.enqueued_at
244
+ end
245
+
246
+ it 'can delete jobs' do
247
+ q = Sidekiq::Queue.new
248
+ ApiWorker.perform_async(1, 'mike')
249
+ assert_equal 1, q.size
250
+
251
+ x = q.first
252
+ assert_equal "TestApi::ApiWorker", x.display_class
253
+ assert_equal [1,'mike'], x.display_args
254
+
255
+ assert_equal [true], q.map(&:delete)
256
+ assert_equal 0, q.size
257
+ end
258
+
259
+ it "can move scheduled job to queue" do
260
+ remain_id = ApiWorker.perform_in(100, 1, 'jason')
261
+ job_id = ApiWorker.perform_in(100, 1, 'jason')
262
+ job = Sidekiq::ScheduledSet.new.find_job(job_id)
263
+ q = Sidekiq::Queue.new
264
+ job.add_to_queue
265
+ queued_job = q.find_job(job_id)
266
+ refute_nil queued_job
267
+ assert_equal queued_job.jid, job_id
268
+ assert_nil Sidekiq::ScheduledSet.new.find_job(job_id)
269
+ refute_nil Sidekiq::ScheduledSet.new.find_job(remain_id)
270
+ end
271
+
272
+ it "handles multiple scheduled jobs when moving to queue" do
273
+ jids = Sidekiq::Client.push_bulk('class' => ApiWorker,
274
+ 'args' => [[1, 'jason'], [2, 'jason']],
275
+ 'at' => Time.now.to_f)
276
+ assert_equal 2, jids.size
277
+ (remain_id, job_id) = jids
278
+ job = Sidekiq::ScheduledSet.new.find_job(job_id)
279
+ q = Sidekiq::Queue.new
280
+ job.add_to_queue
281
+ queued_job = q.find_job(job_id)
282
+ refute_nil queued_job
283
+ assert_equal queued_job.jid, job_id
284
+ assert_nil Sidekiq::ScheduledSet.new.find_job(job_id)
285
+ refute_nil Sidekiq::ScheduledSet.new.find_job(remain_id)
286
+ end
287
+
288
+ it 'can find job by id in sorted sets' do
289
+ job_id = ApiWorker.perform_in(100, 1, 'jason')
290
+ job = Sidekiq::ScheduledSet.new.find_job(job_id)
291
+ refute_nil job
292
+ assert_equal job_id, job.jid
293
+ assert_in_delta job.latency, 0.0, 0.1
294
+ end
295
+
296
+ it 'can remove jobs when iterating over a sorted set' do
297
+ # scheduled jobs must be greater than SortedSet#each underlying page size
298
+ 51.times do
299
+ ApiWorker.perform_in(100, 'aaron')
300
+ end
301
+ set = Sidekiq::ScheduledSet.new
302
+ set.map(&:delete)
303
+ assert_equal set.size, 0
304
+ end
305
+
306
+ it 'can remove jobs when iterating over a queue' do
307
+ # initial queue size must be greater than Queue#each underlying page size
308
+ 51.times do
309
+ ApiWorker.perform_async(1, 'aaron')
310
+ end
311
+ q = Sidekiq::Queue.new
312
+ q.map(&:delete)
313
+ assert_equal q.size, 0
314
+ end
315
+
316
+ it 'can find job by id in queues' do
317
+ q = Sidekiq::Queue.new
318
+ job_id = ApiWorker.perform_async(1, 'jason')
319
+ job = q.find_job(job_id)
320
+ refute_nil job
321
+ assert_equal job_id, job.jid
322
+ end
323
+
324
+ it 'can clear a queue' do
325
+ q = Sidekiq::Queue.new
326
+ 2.times { ApiWorker.perform_async(1, 'mike') }
327
+ q.clear
328
+
329
+ Sidekiq.redis do |conn|
330
+ refute conn.smembers('queues').include?('foo')
331
+ refute conn.exists('queue:foo')
332
+ end
333
+ end
334
+
335
+ it 'can fetch by score' do
336
+ same_time = Time.now.to_f
337
+ add_retry('bob1', same_time)
338
+ add_retry('bob2', same_time)
339
+ r = Sidekiq::RetrySet.new
340
+ assert_equal 2, r.fetch(same_time).size
341
+ end
342
+
343
+ it 'can fetch by score and jid' do
344
+ same_time = Time.now.to_f
345
+ add_retry('bob1', same_time)
346
+ add_retry('bob2', same_time)
347
+ r = Sidekiq::RetrySet.new
348
+ assert_equal 1, r.fetch(same_time, 'bob1').size
349
+ end
350
+
351
+ it 'shows empty retries' do
352
+ r = Sidekiq::RetrySet.new
353
+ assert_equal 0, r.size
354
+ end
355
+
356
+ it 'can enumerate retries' do
357
+ add_retry
358
+
359
+ r = Sidekiq::RetrySet.new
360
+ assert_equal 1, r.size
361
+ array = r.to_a
362
+ assert_equal 1, array.size
363
+
364
+ retri = array.first
365
+ assert_equal 'ApiWorker', retri.klass
366
+ assert_equal 'default', retri.queue
367
+ assert_equal 'bob', retri.jid
368
+ assert_in_delta Time.now.to_f, retri.at.to_f, 0.02
369
+ end
370
+
371
+ it 'requires a jid to delete an entry' do
372
+ start_time = Time.now.to_f
373
+ add_retry('bob2', Time.now.to_f)
374
+ assert_raises(ArgumentError) do
375
+ Sidekiq::RetrySet.new.delete(start_time)
376
+ end
377
+ end
378
+
379
+ it 'can delete a single retry from score and jid' do
380
+ same_time = Time.now.to_f
381
+ add_retry('bob1', same_time)
382
+ add_retry('bob2', same_time)
383
+ r = Sidekiq::RetrySet.new
384
+ assert_equal 2, r.size
385
+ Sidekiq::RetrySet.new.delete(same_time, 'bob1')
386
+ assert_equal 1, r.size
387
+ end
388
+
389
+ it 'can retry a retry' do
390
+ add_retry
391
+ r = Sidekiq::RetrySet.new
392
+ assert_equal 1, r.size
393
+ r.first.retry
394
+ assert_equal 0, r.size
395
+ assert_equal 1, Sidekiq::Queue.new('default').size
396
+ job = Sidekiq::Queue.new('default').first
397
+ assert_equal 'bob', job.jid
398
+ assert_equal 1, job['retry_count']
399
+ end
400
+
401
+ it 'can clear retries' do
402
+ add_retry
403
+ add_retry('test')
404
+ r = Sidekiq::RetrySet.new
405
+ assert_equal 2, r.size
406
+ r.clear
407
+ assert_equal 0, r.size
408
+ end
409
+
410
+ it 'can enumerate processes' do
411
+ identity_string = "identity_string"
412
+ odata = {
413
+ 'pid' => 123,
414
+ 'hostname' => Socket.gethostname,
415
+ 'key' => identity_string,
416
+ 'identity' => identity_string,
417
+ 'started_at' => Time.now.to_f - 15,
418
+ }
419
+
420
+ time = Time.now.to_f
421
+ Sidekiq.redis do |conn|
422
+ conn.multi do
423
+ conn.sadd('processes', odata['key'])
424
+ conn.hmset(odata['key'], 'info', Sidekiq.dump_json(odata), 'busy', 10, 'beat', time)
425
+ conn.sadd('processes', 'fake:pid')
426
+ end
427
+ end
428
+
429
+ ps = Sidekiq::ProcessSet.new.to_a
430
+ assert_equal 1, ps.size
431
+ data = ps.first
432
+ assert_equal 10, data['busy']
433
+ assert_equal time, data['beat']
434
+ assert_equal 123, data['pid']
435
+ data.quiet!
436
+ data.stop!
437
+ signals_string = "#{odata['key']}-signals"
438
+ assert_equal "TERM", Sidekiq.redis{|c| c.lpop(signals_string) }
439
+ assert_equal "USR1", Sidekiq.redis{|c| c.lpop(signals_string) }
440
+ end
441
+
442
+ it 'can enumerate workers' do
443
+ w = Sidekiq::Workers.new
444
+ assert_equal 0, w.size
445
+ w.each do
446
+ assert false
447
+ end
448
+
449
+ hn = Socket.gethostname
450
+ key = "#{hn}:#{$$}"
451
+ pdata = { 'pid' => $$, 'hostname' => hn, 'started_at' => Time.now.to_i }
452
+ Sidekiq.redis do |conn|
453
+ conn.sadd('processes', key)
454
+ conn.hmset(key, 'info', Sidekiq.dump_json(pdata), 'busy', 0, 'beat', Time.now.to_f)
455
+ end
456
+
457
+ s = "#{key}:workers"
458
+ data = Sidekiq.dump_json({ 'payload' => {}, 'queue' => 'default', 'run_at' => Time.now.to_i })
459
+ Sidekiq.redis do |c|
460
+ c.hmset(s, '1234', data)
461
+ end
462
+
463
+ w.each do |p, x, y|
464
+ assert_equal key, p
465
+ assert_equal "1234", x
466
+ assert_equal 'default', y['queue']
467
+ assert_equal Time.now.year, Time.at(y['run_at']).year
468
+ end
469
+
470
+ s = "#{key}:workers"
471
+ data = Sidekiq.dump_json({ 'payload' => {}, 'queue' => 'default', 'run_at' => (Time.now.to_i - 2*60*60) })
472
+ Sidekiq.redis do |c|
473
+ c.multi do
474
+ c.hmset(s, '5678', data)
475
+ c.hmset("b#{s}", '5678', data)
476
+ end
477
+ end
478
+
479
+ assert_equal ['1234', '5678'], w.map { |_, tid, _| tid }
480
+ end
481
+
482
+ it 'can reschedule jobs' do
483
+ add_retry('foo1')
484
+ add_retry('foo2')
485
+
486
+ retries = Sidekiq::RetrySet.new
487
+ assert_equal 2, retries.size
488
+ refute(retries.map { |r| r.score > (Time.now.to_f + 9) }.any?)
489
+
490
+ retries.each do |retri|
491
+ retri.reschedule(Time.now.to_f + 10) if retri.jid == 'foo2'
492
+ end
493
+
494
+ assert_equal 2, retries.size
495
+ assert(retries.map { |r| r.score > (Time.now.to_f + 9) }.any?)
496
+ end
497
+
498
+ it 'prunes processes which have died' do
499
+ data = { 'pid' => rand(10_000), 'hostname' => "app#{rand(1_000)}", 'started_at' => Time.now.to_f }
500
+ key = "#{data['hostname']}:#{data['pid']}"
501
+ Sidekiq.redis do |conn|
502
+ conn.sadd('processes', key)
503
+ conn.hmset(key, 'info', Sidekiq.dump_json(data), 'busy', 0, 'beat', Time.now.to_f)
504
+ end
505
+
506
+ ps = Sidekiq::ProcessSet.new
507
+ assert_equal 1, ps.size
508
+ assert_equal 1, ps.to_a.size
509
+
510
+ Sidekiq.redis do |conn|
511
+ conn.sadd('processes', "bar:987")
512
+ conn.sadd('processes', "bar:986")
513
+ end
514
+
515
+ ps = Sidekiq::ProcessSet.new
516
+ assert_equal 1, ps.size
517
+ assert_equal 1, ps.to_a.size
518
+ end
519
+
520
+ def add_retry(jid = 'bob', at = Time.now.to_f)
521
+ payload = Sidekiq.dump_json('class' => 'ApiWorker', 'args' => [1, 'mike'], 'queue' => 'default', 'jid' => jid, 'retry_count' => 2, 'failed_at' => Time.now.to_f)
522
+ Sidekiq.redis do |conn|
523
+ conn.zadd('retry', at.to_s, payload)
524
+ end
525
+ end
526
+ end
527
+ end
528
+ end