sidekiq 5.2.6

Sign up to get free protection for your applications and to get access to all the features.

Potentially problematic release.


This version of sidekiq might be problematic. Click here for more details.

Files changed (118) hide show
  1. checksums.yaml +7 -0
  2. data/.github/contributing.md +32 -0
  3. data/.github/issue_template.md +11 -0
  4. data/.gitignore +15 -0
  5. data/.travis.yml +11 -0
  6. data/3.0-Upgrade.md +70 -0
  7. data/4.0-Upgrade.md +53 -0
  8. data/5.0-Upgrade.md +56 -0
  9. data/COMM-LICENSE +97 -0
  10. data/Changes.md +1529 -0
  11. data/Ent-Changes.md +238 -0
  12. data/Gemfile +23 -0
  13. data/LICENSE +9 -0
  14. data/Pro-2.0-Upgrade.md +138 -0
  15. data/Pro-3.0-Upgrade.md +44 -0
  16. data/Pro-4.0-Upgrade.md +35 -0
  17. data/Pro-Changes.md +759 -0
  18. data/README.md +109 -0
  19. data/Rakefile +9 -0
  20. data/bin/sidekiq +18 -0
  21. data/bin/sidekiqctl +20 -0
  22. data/bin/sidekiqload +149 -0
  23. data/code_of_conduct.md +50 -0
  24. data/lib/generators/sidekiq/templates/worker.rb.erb +9 -0
  25. data/lib/generators/sidekiq/templates/worker_spec.rb.erb +6 -0
  26. data/lib/generators/sidekiq/templates/worker_test.rb.erb +8 -0
  27. data/lib/generators/sidekiq/worker_generator.rb +49 -0
  28. data/lib/sidekiq.rb +237 -0
  29. data/lib/sidekiq/api.rb +940 -0
  30. data/lib/sidekiq/cli.rb +445 -0
  31. data/lib/sidekiq/client.rb +243 -0
  32. data/lib/sidekiq/core_ext.rb +1 -0
  33. data/lib/sidekiq/ctl.rb +221 -0
  34. data/lib/sidekiq/delay.rb +42 -0
  35. data/lib/sidekiq/exception_handler.rb +29 -0
  36. data/lib/sidekiq/extensions/action_mailer.rb +57 -0
  37. data/lib/sidekiq/extensions/active_record.rb +40 -0
  38. data/lib/sidekiq/extensions/class_methods.rb +40 -0
  39. data/lib/sidekiq/extensions/generic_proxy.rb +31 -0
  40. data/lib/sidekiq/fetch.rb +81 -0
  41. data/lib/sidekiq/job_logger.rb +25 -0
  42. data/lib/sidekiq/job_retry.rb +262 -0
  43. data/lib/sidekiq/launcher.rb +173 -0
  44. data/lib/sidekiq/logging.rb +122 -0
  45. data/lib/sidekiq/manager.rb +137 -0
  46. data/lib/sidekiq/middleware/chain.rb +150 -0
  47. data/lib/sidekiq/middleware/i18n.rb +42 -0
  48. data/lib/sidekiq/middleware/server/active_record.rb +23 -0
  49. data/lib/sidekiq/paginator.rb +43 -0
  50. data/lib/sidekiq/processor.rb +279 -0
  51. data/lib/sidekiq/rails.rb +58 -0
  52. data/lib/sidekiq/redis_connection.rb +144 -0
  53. data/lib/sidekiq/scheduled.rb +174 -0
  54. data/lib/sidekiq/testing.rb +333 -0
  55. data/lib/sidekiq/testing/inline.rb +29 -0
  56. data/lib/sidekiq/util.rb +66 -0
  57. data/lib/sidekiq/version.rb +4 -0
  58. data/lib/sidekiq/web.rb +213 -0
  59. data/lib/sidekiq/web/action.rb +89 -0
  60. data/lib/sidekiq/web/application.rb +353 -0
  61. data/lib/sidekiq/web/helpers.rb +325 -0
  62. data/lib/sidekiq/web/router.rb +100 -0
  63. data/lib/sidekiq/worker.rb +220 -0
  64. data/sidekiq.gemspec +21 -0
  65. data/web/assets/images/favicon.ico +0 -0
  66. data/web/assets/images/logo.png +0 -0
  67. data/web/assets/images/status.png +0 -0
  68. data/web/assets/javascripts/application.js +92 -0
  69. data/web/assets/javascripts/dashboard.js +315 -0
  70. data/web/assets/stylesheets/application-rtl.css +246 -0
  71. data/web/assets/stylesheets/application.css +1144 -0
  72. data/web/assets/stylesheets/bootstrap-rtl.min.css +9 -0
  73. data/web/assets/stylesheets/bootstrap.css +5 -0
  74. data/web/locales/ar.yml +81 -0
  75. data/web/locales/cs.yml +78 -0
  76. data/web/locales/da.yml +68 -0
  77. data/web/locales/de.yml +69 -0
  78. data/web/locales/el.yml +68 -0
  79. data/web/locales/en.yml +81 -0
  80. data/web/locales/es.yml +70 -0
  81. data/web/locales/fa.yml +80 -0
  82. data/web/locales/fr.yml +78 -0
  83. data/web/locales/he.yml +79 -0
  84. data/web/locales/hi.yml +75 -0
  85. data/web/locales/it.yml +69 -0
  86. data/web/locales/ja.yml +80 -0
  87. data/web/locales/ko.yml +68 -0
  88. data/web/locales/nb.yml +77 -0
  89. data/web/locales/nl.yml +68 -0
  90. data/web/locales/pl.yml +59 -0
  91. data/web/locales/pt-br.yml +68 -0
  92. data/web/locales/pt.yml +67 -0
  93. data/web/locales/ru.yml +78 -0
  94. data/web/locales/sv.yml +68 -0
  95. data/web/locales/ta.yml +75 -0
  96. data/web/locales/uk.yml +76 -0
  97. data/web/locales/ur.yml +80 -0
  98. data/web/locales/zh-cn.yml +68 -0
  99. data/web/locales/zh-tw.yml +68 -0
  100. data/web/views/_footer.erb +20 -0
  101. data/web/views/_job_info.erb +88 -0
  102. data/web/views/_nav.erb +52 -0
  103. data/web/views/_paging.erb +23 -0
  104. data/web/views/_poll_link.erb +7 -0
  105. data/web/views/_status.erb +4 -0
  106. data/web/views/_summary.erb +40 -0
  107. data/web/views/busy.erb +98 -0
  108. data/web/views/dashboard.erb +75 -0
  109. data/web/views/dead.erb +34 -0
  110. data/web/views/layout.erb +40 -0
  111. data/web/views/morgue.erb +75 -0
  112. data/web/views/queue.erb +46 -0
  113. data/web/views/queues.erb +30 -0
  114. data/web/views/retries.erb +80 -0
  115. data/web/views/retry.erb +34 -0
  116. data/web/views/scheduled.erb +54 -0
  117. data/web/views/scheduled_job_info.erb +8 -0
  118. metadata +229 -0
@@ -0,0 +1,122 @@
1
+ # frozen_string_literal: true
2
+ require 'time'
3
+ require 'logger'
4
+ require 'fcntl'
5
+
6
+ module Sidekiq
7
+ module Logging
8
+
9
+ class Pretty < Logger::Formatter
10
+ SPACE = " "
11
+
12
+ # Provide a call() method that returns the formatted message.
13
+ def call(severity, time, program_name, message)
14
+ "#{time.utc.iso8601(3)} #{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
15
+ end
16
+
17
+ def context
18
+ c = Thread.current[:sidekiq_context]
19
+ " #{c.join(SPACE)}" if c && c.any?
20
+ end
21
+ end
22
+
23
+ class WithoutTimestamp < Pretty
24
+ def call(severity, time, program_name, message)
25
+ "#{::Process.pid} TID-#{Sidekiq::Logging.tid}#{context} #{severity}: #{message}\n"
26
+ end
27
+ end
28
+
29
+ def self.tid
30
+ Thread.current['sidekiq_tid'] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
31
+ end
32
+
33
+ def self.job_hash_context(job_hash)
34
+ # If we're using a wrapper class, like ActiveJob, use the "wrapped"
35
+ # attribute to expose the underlying thing.
36
+ klass = job_hash['wrapped'] || job_hash["class"]
37
+ bid = job_hash['bid']
38
+ "#{klass} JID-#{job_hash['jid']}#{" BID-#{bid}" if bid}"
39
+ end
40
+
41
+ def self.with_job_hash_context(job_hash, &block)
42
+ with_context(job_hash_context(job_hash), &block)
43
+ end
44
+
45
+ def self.with_context(msg)
46
+ Thread.current[:sidekiq_context] ||= []
47
+ Thread.current[:sidekiq_context] << msg
48
+ yield
49
+ ensure
50
+ Thread.current[:sidekiq_context].pop
51
+ end
52
+
53
+ def self.initialize_logger(log_target = STDOUT)
54
+ oldlogger = defined?(@logger) ? @logger : nil
55
+ @logger = Logger.new(log_target)
56
+ @logger.level = Logger::INFO
57
+ @logger.formatter = ENV['DYNO'] ? WithoutTimestamp.new : Pretty.new
58
+ oldlogger.close if oldlogger && !$TESTING # don't want to close testing's STDOUT logging
59
+ @logger
60
+ end
61
+
62
+ def self.logger
63
+ defined?(@logger) ? @logger : initialize_logger
64
+ end
65
+
66
+ def self.logger=(log)
67
+ @logger = (log ? log : Logger.new(File::NULL))
68
+ end
69
+
70
+ # This reopens ALL logfiles in the process that have been rotated
71
+ # using logrotate(8) (without copytruncate) or similar tools.
72
+ # A +File+ object is considered for reopening if it is:
73
+ # 1) opened with the O_APPEND and O_WRONLY flags
74
+ # 2) the current open file handle does not match its original open path
75
+ # 3) unbuffered (as far as userspace buffering goes, not O_SYNC)
76
+ # Returns the number of files reopened
77
+ def self.reopen_logs
78
+ to_reopen = []
79
+ append_flags = File::WRONLY | File::APPEND
80
+
81
+ ObjectSpace.each_object(File) do |fp|
82
+ begin
83
+ if !fp.closed? && fp.stat.file? && fp.sync && (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
84
+ to_reopen << fp
85
+ end
86
+ rescue IOError, Errno::EBADF
87
+ end
88
+ end
89
+
90
+ nr = 0
91
+ to_reopen.each do |fp|
92
+ orig_st = begin
93
+ fp.stat
94
+ rescue IOError, Errno::EBADF
95
+ next
96
+ end
97
+
98
+ begin
99
+ b = File.stat(fp.path)
100
+ next if orig_st.ino == b.ino && orig_st.dev == b.dev
101
+ rescue Errno::ENOENT
102
+ end
103
+
104
+ begin
105
+ File.open(fp.path, 'a') { |tmpfp| fp.reopen(tmpfp) }
106
+ fp.sync = true
107
+ nr += 1
108
+ rescue IOError, Errno::EBADF
109
+ # not much we can do...
110
+ end
111
+ end
112
+ nr
113
+ rescue RuntimeError => ex
114
+ # RuntimeError: ObjectSpace is disabled; each_object will only work with Class, pass -X+O to enable
115
+ puts "Unable to reopen logs: #{ex.message}"
116
+ end
117
+
118
+ def logger
119
+ Sidekiq::Logging.logger
120
+ end
121
+ end
122
+ end
@@ -0,0 +1,137 @@
1
+ # frozen_string_literal: true
2
+ require 'sidekiq/util'
3
+ require 'sidekiq/processor'
4
+ require 'sidekiq/fetch'
5
+ require 'thread'
6
+ require 'set'
7
+
8
+ module Sidekiq
9
+
10
+ ##
11
+ # The Manager is the central coordination point in Sidekiq, controlling
12
+ # the lifecycle of the Processors.
13
+ #
14
+ # Tasks:
15
+ #
16
+ # 1. start: Spin up Processors.
17
+ # 3. processor_died: Handle job failure, throw away Processor, create new one.
18
+ # 4. quiet: shutdown idle Processors.
19
+ # 5. stop: hard stop the Processors by deadline.
20
+ #
21
+ # Note that only the last task requires its own Thread since it has to monitor
22
+ # the shutdown process. The other tasks are performed by other threads.
23
+ #
24
+ class Manager
25
+ include Util
26
+
27
+ attr_reader :workers
28
+ attr_reader :options
29
+
30
+ def initialize(options={})
31
+ logger.debug { options.inspect }
32
+ @options = options
33
+ @count = options[:concurrency] || 10
34
+ raise ArgumentError, "Concurrency of #{@count} is not supported" if @count < 1
35
+
36
+ @done = false
37
+ @workers = Set.new
38
+ @count.times do
39
+ @workers << Processor.new(self)
40
+ end
41
+ @plock = Mutex.new
42
+ end
43
+
44
+ def start
45
+ @workers.each do |x|
46
+ x.start
47
+ end
48
+ end
49
+
50
+ def quiet
51
+ return if @done
52
+ @done = true
53
+
54
+ logger.info { "Terminating quiet workers" }
55
+ @workers.each { |x| x.terminate }
56
+ fire_event(:quiet, reverse: true)
57
+ end
58
+
59
+ # hack for quicker development / testing environment #2774
60
+ PAUSE_TIME = STDOUT.tty? ? 0.1 : 0.5
61
+
62
+ def stop(deadline)
63
+ quiet
64
+ fire_event(:shutdown, reverse: true)
65
+
66
+ # some of the shutdown events can be async,
67
+ # we don't have any way to know when they're done but
68
+ # give them a little time to take effect
69
+ sleep PAUSE_TIME
70
+ return if @workers.empty?
71
+
72
+ logger.info { "Pausing to allow workers to finish..." }
73
+ remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
74
+ while remaining > PAUSE_TIME
75
+ return if @workers.empty?
76
+ sleep PAUSE_TIME
77
+ remaining = deadline - ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
78
+ end
79
+ return if @workers.empty?
80
+
81
+ hard_shutdown
82
+ end
83
+
84
+ def processor_stopped(processor)
85
+ @plock.synchronize do
86
+ @workers.delete(processor)
87
+ end
88
+ end
89
+
90
+ def processor_died(processor, reason)
91
+ @plock.synchronize do
92
+ @workers.delete(processor)
93
+ unless @done
94
+ p = Processor.new(self)
95
+ @workers << p
96
+ p.start
97
+ end
98
+ end
99
+ end
100
+
101
+ def stopped?
102
+ @done
103
+ end
104
+
105
+ private
106
+
107
+ def hard_shutdown
108
+ # We've reached the timeout and we still have busy workers.
109
+ # They must die but their jobs shall live on.
110
+ cleanup = nil
111
+ @plock.synchronize do
112
+ cleanup = @workers.dup
113
+ end
114
+
115
+ if cleanup.size > 0
116
+ jobs = cleanup.map {|p| p.job }.compact
117
+
118
+ logger.warn { "Terminating #{cleanup.size} busy worker threads" }
119
+ logger.warn { "Work still in progress #{jobs.inspect}" }
120
+
121
+ # Re-enqueue unfinished jobs
122
+ # NOTE: You may notice that we may push a job back to redis before
123
+ # the worker thread is terminated. This is ok because Sidekiq's
124
+ # contract says that jobs are run AT LEAST once. Process termination
125
+ # is delayed until we're certain the jobs are back in Redis because
126
+ # it is worse to lose a job than to run it twice.
127
+ strategy = (@options[:fetch] || Sidekiq::BasicFetch)
128
+ strategy.bulk_requeue(jobs, @options)
129
+ end
130
+
131
+ cleanup.each do |processor|
132
+ processor.kill
133
+ end
134
+ end
135
+
136
+ end
137
+ end
@@ -0,0 +1,150 @@
1
+ # frozen_string_literal: true
2
+ module Sidekiq
3
+ # Middleware is code configured to run before/after
4
+ # a message is processed. It is patterned after Rack
5
+ # middleware. Middleware exists for the client side
6
+ # (pushing jobs onto the queue) as well as the server
7
+ # side (when jobs are actually processed).
8
+ #
9
+ # To add middleware for the client:
10
+ #
11
+ # Sidekiq.configure_client do |config|
12
+ # config.client_middleware do |chain|
13
+ # chain.add MyClientHook
14
+ # end
15
+ # end
16
+ #
17
+ # To modify middleware for the server, just call
18
+ # with another block:
19
+ #
20
+ # Sidekiq.configure_server do |config|
21
+ # config.server_middleware do |chain|
22
+ # chain.add MyServerHook
23
+ # chain.remove ActiveRecord
24
+ # end
25
+ # end
26
+ #
27
+ # To insert immediately preceding another entry:
28
+ #
29
+ # Sidekiq.configure_client do |config|
30
+ # config.client_middleware do |chain|
31
+ # chain.insert_before ActiveRecord, MyClientHook
32
+ # end
33
+ # end
34
+ #
35
+ # To insert immediately after another entry:
36
+ #
37
+ # Sidekiq.configure_client do |config|
38
+ # config.client_middleware do |chain|
39
+ # chain.insert_after ActiveRecord, MyClientHook
40
+ # end
41
+ # end
42
+ #
43
+ # This is an example of a minimal server middleware:
44
+ #
45
+ # class MyServerHook
46
+ # def call(worker_instance, msg, queue)
47
+ # puts "Before work"
48
+ # yield
49
+ # puts "After work"
50
+ # end
51
+ # end
52
+ #
53
+ # This is an example of a minimal client middleware, note
54
+ # the method must return the result or the job will not push
55
+ # to Redis:
56
+ #
57
+ # class MyClientHook
58
+ # def call(worker_class, msg, queue, redis_pool)
59
+ # puts "Before push"
60
+ # result = yield
61
+ # puts "After push"
62
+ # result
63
+ # end
64
+ # end
65
+ #
66
+ module Middleware
67
+ class Chain
68
+ include Enumerable
69
+ attr_reader :entries
70
+
71
+ def initialize_copy(copy)
72
+ copy.instance_variable_set(:@entries, entries.dup)
73
+ end
74
+
75
+ def each(&block)
76
+ entries.each(&block)
77
+ end
78
+
79
+ def initialize
80
+ @entries = []
81
+ yield self if block_given?
82
+ end
83
+
84
+ def remove(klass)
85
+ entries.delete_if { |entry| entry.klass == klass }
86
+ end
87
+
88
+ def add(klass, *args)
89
+ remove(klass) if exists?(klass)
90
+ entries << Entry.new(klass, *args)
91
+ end
92
+
93
+ def prepend(klass, *args)
94
+ remove(klass) if exists?(klass)
95
+ entries.insert(0, Entry.new(klass, *args))
96
+ end
97
+
98
+ def insert_before(oldklass, newklass, *args)
99
+ i = entries.index { |entry| entry.klass == newklass }
100
+ new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
101
+ i = entries.index { |entry| entry.klass == oldklass } || 0
102
+ entries.insert(i, new_entry)
103
+ end
104
+
105
+ def insert_after(oldklass, newklass, *args)
106
+ i = entries.index { |entry| entry.klass == newklass }
107
+ new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
108
+ i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
109
+ entries.insert(i+1, new_entry)
110
+ end
111
+
112
+ def exists?(klass)
113
+ any? { |entry| entry.klass == klass }
114
+ end
115
+
116
+ def retrieve
117
+ map(&:make_new)
118
+ end
119
+
120
+ def clear
121
+ entries.clear
122
+ end
123
+
124
+ def invoke(*args)
125
+ chain = retrieve.dup
126
+ traverse_chain = lambda do
127
+ if chain.empty?
128
+ yield
129
+ else
130
+ chain.shift.call(*args, &traverse_chain)
131
+ end
132
+ end
133
+ traverse_chain.call
134
+ end
135
+ end
136
+
137
+ class Entry
138
+ attr_reader :klass
139
+
140
+ def initialize(klass, *args)
141
+ @klass = klass
142
+ @args = args
143
+ end
144
+
145
+ def make_new
146
+ @klass.new(*@args)
147
+ end
148
+ end
149
+ end
150
+ end
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+ #
3
+ # Simple middleware to save the current locale and restore it when the job executes.
4
+ # Use it by requiring it in your initializer:
5
+ #
6
+ # require 'sidekiq/middleware/i18n'
7
+ #
8
+ module Sidekiq::Middleware::I18n
9
+ # Get the current locale and store it in the message
10
+ # to be sent to Sidekiq.
11
+ class Client
12
+ def call(worker_class, msg, queue, redis_pool)
13
+ msg['locale'] ||= I18n.locale
14
+ yield
15
+ end
16
+ end
17
+
18
+ # Pull the msg locale out and set the current thread to use it.
19
+ class Server
20
+ def call(worker, msg, queue)
21
+ I18n.locale = msg['locale'] || I18n.default_locale
22
+ yield
23
+ ensure
24
+ I18n.locale = I18n.default_locale
25
+ end
26
+ end
27
+ end
28
+
29
+ Sidekiq.configure_client do |config|
30
+ config.client_middleware do |chain|
31
+ chain.add Sidekiq::Middleware::I18n::Client
32
+ end
33
+ end
34
+
35
+ Sidekiq.configure_server do |config|
36
+ config.client_middleware do |chain|
37
+ chain.add Sidekiq::Middleware::I18n::Client
38
+ end
39
+ config.server_middleware do |chain|
40
+ chain.add Sidekiq::Middleware::I18n::Server
41
+ end
42
+ end