sr-sidekiq 4.1.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/3.0-Upgrade.md +70 -0
- data/4.0-Upgrade.md +50 -0
- data/COMM-LICENSE (sidekiq) +95 -0
- data/Changes.md +1241 -0
- data/Ent-Changes.md +112 -0
- data/Gemfile +29 -0
- data/LICENSE (sidekiq) +9 -0
- data/LICENSE (sr-sidekiq) +5 -0
- data/Pro-2.0-Upgrade.md +138 -0
- data/Pro-3.0-Upgrade.md +44 -0
- data/Pro-Changes.md +539 -0
- data/README.md +8 -0
- data/Rakefile +9 -0
- data/bin/sidekiq +18 -0
- data/bin/sidekiqctl +99 -0
- data/bin/sidekiqload +167 -0
- data/code_of_conduct.md +50 -0
- data/lib/generators/sidekiq/templates/worker.rb.erb +9 -0
- data/lib/generators/sidekiq/templates/worker_spec.rb.erb +6 -0
- data/lib/generators/sidekiq/templates/worker_test.rb.erb +8 -0
- data/lib/generators/sidekiq/worker_generator.rb +49 -0
- data/lib/sidekiq.rb +237 -0
- data/lib/sidekiq/api.rb +844 -0
- data/lib/sidekiq/cli.rb +389 -0
- data/lib/sidekiq/client.rb +260 -0
- data/lib/sidekiq/core_ext.rb +106 -0
- data/lib/sidekiq/exception_handler.rb +31 -0
- data/lib/sidekiq/extensions/action_mailer.rb +57 -0
- data/lib/sidekiq/extensions/active_record.rb +40 -0
- data/lib/sidekiq/extensions/class_methods.rb +40 -0
- data/lib/sidekiq/extensions/generic_proxy.rb +25 -0
- data/lib/sidekiq/fetch.rb +81 -0
- data/lib/sidekiq/launcher.rb +160 -0
- data/lib/sidekiq/logging.rb +106 -0
- data/lib/sidekiq/manager.rb +137 -0
- data/lib/sidekiq/middleware/chain.rb +150 -0
- data/lib/sidekiq/middleware/i18n.rb +42 -0
- data/lib/sidekiq/middleware/server/active_record.rb +13 -0
- data/lib/sidekiq/middleware/server/logging.rb +40 -0
- data/lib/sidekiq/middleware/server/retry_jobs.rb +205 -0
- data/lib/sidekiq/paginator.rb +43 -0
- data/lib/sidekiq/processor.rb +186 -0
- data/lib/sidekiq/rails.rb +39 -0
- data/lib/sidekiq/redis_connection.rb +97 -0
- data/lib/sidekiq/scheduled.rb +146 -0
- data/lib/sidekiq/testing.rb +316 -0
- data/lib/sidekiq/testing/inline.rb +29 -0
- data/lib/sidekiq/util.rb +62 -0
- data/lib/sidekiq/version.rb +4 -0
- data/lib/sidekiq/web.rb +278 -0
- data/lib/sidekiq/web_helpers.rb +255 -0
- data/lib/sidekiq/worker.rb +121 -0
- data/sidekiq.gemspec +26 -0
- data/sr-sidekiq-4.1.3.gem +0 -0
- data/sr-sidekiq-4.1.4.gem +0 -0
- data/sr-sidekiq-4.1.5.gem +0 -0
- data/test/config.yml +9 -0
- data/test/env_based_config.yml +11 -0
- data/test/fake_env.rb +1 -0
- data/test/fixtures/en.yml +2 -0
- data/test/helper.rb +75 -0
- data/test/test_actors.rb +138 -0
- data/test/test_api.rb +528 -0
- data/test/test_cli.rb +406 -0
- data/test/test_client.rb +262 -0
- data/test/test_exception_handler.rb +56 -0
- data/test/test_extensions.rb +127 -0
- data/test/test_fetch.rb +50 -0
- data/test/test_launcher.rb +85 -0
- data/test/test_logging.rb +35 -0
- data/test/test_manager.rb +50 -0
- data/test/test_middleware.rb +158 -0
- data/test/test_processor.rb +201 -0
- data/test/test_rails.rb +22 -0
- data/test/test_redis_connection.rb +127 -0
- data/test/test_retry.rb +326 -0
- data/test/test_retry_exhausted.rb +149 -0
- data/test/test_scheduled.rb +115 -0
- data/test/test_scheduling.rb +50 -0
- data/test/test_sidekiq.rb +107 -0
- data/test/test_testing.rb +143 -0
- data/test/test_testing_fake.rb +357 -0
- data/test/test_testing_inline.rb +94 -0
- data/test/test_util.rb +13 -0
- data/test/test_web.rb +614 -0
- data/test/test_web_helpers.rb +54 -0
- data/web/assets/images/bootstrap/glyphicons-halflings-white.png +0 -0
- data/web/assets/images/bootstrap/glyphicons-halflings.png +0 -0
- data/web/assets/images/favicon.ico +0 -0
- data/web/assets/images/logo.png +0 -0
- data/web/assets/images/status-sd8051fd480.png +0 -0
- data/web/assets/images/status/active.png +0 -0
- data/web/assets/images/status/idle.png +0 -0
- data/web/assets/javascripts/application.js +88 -0
- data/web/assets/javascripts/dashboard.js +300 -0
- data/web/assets/javascripts/locales/README.md +27 -0
- data/web/assets/javascripts/locales/jquery.timeago.ar.js +96 -0
- data/web/assets/javascripts/locales/jquery.timeago.bg.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.bs.js +49 -0
- data/web/assets/javascripts/locales/jquery.timeago.ca.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.cs.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.cy.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.da.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.de.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.el.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.en-short.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.en.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.es.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.et.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.fa.js +22 -0
- data/web/assets/javascripts/locales/jquery.timeago.fi.js +28 -0
- data/web/assets/javascripts/locales/jquery.timeago.fr-short.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.fr.js +17 -0
- data/web/assets/javascripts/locales/jquery.timeago.he.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.hr.js +49 -0
- data/web/assets/javascripts/locales/jquery.timeago.hu.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.hy.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.id.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.it.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.ja.js +19 -0
- data/web/assets/javascripts/locales/jquery.timeago.ko.js +17 -0
- data/web/assets/javascripts/locales/jquery.timeago.lt.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.mk.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.nl.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.no.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.pl.js +31 -0
- data/web/assets/javascripts/locales/jquery.timeago.pt-br.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.pt.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.ro.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.rs.js +49 -0
- data/web/assets/javascripts/locales/jquery.timeago.ru.js +34 -0
- data/web/assets/javascripts/locales/jquery.timeago.sk.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.sl.js +44 -0
- data/web/assets/javascripts/locales/jquery.timeago.sv.js +18 -0
- data/web/assets/javascripts/locales/jquery.timeago.th.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.tr.js +16 -0
- data/web/assets/javascripts/locales/jquery.timeago.uk.js +34 -0
- data/web/assets/javascripts/locales/jquery.timeago.uz.js +19 -0
- data/web/assets/javascripts/locales/jquery.timeago.zh-cn.js +20 -0
- data/web/assets/javascripts/locales/jquery.timeago.zh-tw.js +20 -0
- data/web/assets/stylesheets/application.css +754 -0
- data/web/assets/stylesheets/bootstrap.css +9 -0
- data/web/locales/cs.yml +78 -0
- data/web/locales/da.yml +68 -0
- data/web/locales/de.yml +69 -0
- data/web/locales/el.yml +68 -0
- data/web/locales/en.yml +79 -0
- data/web/locales/es.yml +69 -0
- data/web/locales/fr.yml +78 -0
- data/web/locales/hi.yml +75 -0
- data/web/locales/it.yml +69 -0
- data/web/locales/ja.yml +78 -0
- data/web/locales/ko.yml +68 -0
- data/web/locales/nb.yml +77 -0
- data/web/locales/nl.yml +68 -0
- data/web/locales/pl.yml +59 -0
- data/web/locales/pt-br.yml +68 -0
- data/web/locales/pt.yml +67 -0
- data/web/locales/ru.yml +78 -0
- data/web/locales/sv.yml +68 -0
- data/web/locales/ta.yml +75 -0
- data/web/locales/uk.yml +76 -0
- data/web/locales/zh-cn.yml +68 -0
- data/web/locales/zh-tw.yml +68 -0
- data/web/views/_footer.erb +17 -0
- data/web/views/_job_info.erb +88 -0
- data/web/views/_nav.erb +66 -0
- data/web/views/_paging.erb +23 -0
- data/web/views/_poll_js.erb +5 -0
- data/web/views/_poll_link.erb +7 -0
- data/web/views/_status.erb +4 -0
- data/web/views/_summary.erb +40 -0
- data/web/views/busy.erb +94 -0
- data/web/views/dashboard.erb +75 -0
- data/web/views/dead.erb +34 -0
- data/web/views/layout.erb +32 -0
- data/web/views/morgue.erb +71 -0
- data/web/views/queue.erb +45 -0
- data/web/views/queues.erb +28 -0
- data/web/views/retries.erb +74 -0
- data/web/views/retry.erb +34 -0
- data/web/views/scheduled.erb +54 -0
- data/web/views/scheduled_job_info.erb +8 -0
- metadata +408 -0
data/lib/sidekiq/cli.rb
ADDED
@@ -0,0 +1,389 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
# encoding: utf-8
|
3
|
+
$stdout.sync = true
|
4
|
+
|
5
|
+
require 'yaml'
|
6
|
+
require 'singleton'
|
7
|
+
require 'optparse'
|
8
|
+
require 'erb'
|
9
|
+
require 'fileutils'
|
10
|
+
|
11
|
+
require 'sidekiq'
|
12
|
+
require 'sidekiq/util'
|
13
|
+
|
14
|
+
module Sidekiq
|
15
|
+
class CLI
|
16
|
+
include Util
|
17
|
+
include Singleton unless $TESTING
|
18
|
+
|
19
|
+
PROCTITLES = [
|
20
|
+
proc { 'sidekiq'.freeze },
|
21
|
+
proc { Sidekiq::VERSION },
|
22
|
+
proc { |me, data| data['tag'] },
|
23
|
+
proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data['concurrency']} busy]" },
|
24
|
+
proc { |me, data| "stopping" if me.stopping? },
|
25
|
+
]
|
26
|
+
|
27
|
+
# Used for CLI testing
|
28
|
+
attr_accessor :code
|
29
|
+
attr_accessor :launcher
|
30
|
+
attr_accessor :environment
|
31
|
+
|
32
|
+
def initialize
|
33
|
+
@code = nil
|
34
|
+
end
|
35
|
+
|
36
|
+
def parse(args=ARGV)
|
37
|
+
@code = nil
|
38
|
+
|
39
|
+
setup_options(args)
|
40
|
+
initialize_logger
|
41
|
+
validate!
|
42
|
+
daemonize
|
43
|
+
write_pid
|
44
|
+
end
|
45
|
+
|
46
|
+
# Code within this method is not tested because it alters
|
47
|
+
# global process state irreversibly. PRs which improve the
|
48
|
+
# test coverage of Sidekiq::CLI are welcomed.
|
49
|
+
def run
|
50
|
+
boot_system
|
51
|
+
print_banner
|
52
|
+
|
53
|
+
self_read, self_write = IO.pipe
|
54
|
+
|
55
|
+
%w(INT TERM USR1 USR2 TTIN).each do |sig|
|
56
|
+
begin
|
57
|
+
trap sig do
|
58
|
+
self_write.puts(sig)
|
59
|
+
end
|
60
|
+
rescue ArgumentError
|
61
|
+
puts "Signal #{sig} not supported"
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
logger.info "Running in #{RUBY_DESCRIPTION}"
|
66
|
+
logger.info Sidekiq::LICENSE
|
67
|
+
logger.info "Upgrade to Sidekiq Pro for more features and support: http://sidekiq.org" unless defined?(::Sidekiq::Pro)
|
68
|
+
|
69
|
+
# touch the connection pool so it is created before we
|
70
|
+
# fire startup and start multithreading.
|
71
|
+
ver = Sidekiq.redis_info['redis_version']
|
72
|
+
raise "You are using Redis v#{ver}, Sidekiq requires Redis v2.8.0 or greater" if ver < '2.8'
|
73
|
+
|
74
|
+
# Before this point, the process is initializing with just the main thread.
|
75
|
+
# Starting here the process will now have multiple threads running.
|
76
|
+
fire_event(:startup)
|
77
|
+
|
78
|
+
logger.debug { "Client Middleware: #{Sidekiq.client_middleware.map(&:klass).join(', ')}" }
|
79
|
+
logger.debug { "Server Middleware: #{Sidekiq.server_middleware.map(&:klass).join(', ')}" }
|
80
|
+
|
81
|
+
if !options[:daemon]
|
82
|
+
logger.info 'Starting processing, hit Ctrl-C to stop'
|
83
|
+
end
|
84
|
+
|
85
|
+
require 'sidekiq/launcher'
|
86
|
+
@launcher = Sidekiq::Launcher.new(options)
|
87
|
+
|
88
|
+
begin
|
89
|
+
launcher.run
|
90
|
+
|
91
|
+
while readable_io = IO.select([self_read])
|
92
|
+
signal = readable_io.first[0].gets.strip
|
93
|
+
handle_signal(signal)
|
94
|
+
end
|
95
|
+
rescue Interrupt
|
96
|
+
logger.info 'Shutting down'
|
97
|
+
launcher.stop
|
98
|
+
# Explicitly exit so busy Processor threads can't block
|
99
|
+
# process shutdown.
|
100
|
+
logger.info "Bye!"
|
101
|
+
exit(0)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def self.banner
|
106
|
+
%q{
|
107
|
+
m,
|
108
|
+
`$b
|
109
|
+
.ss, $$: .,d$
|
110
|
+
`$$P,d$P' .,md$P"'
|
111
|
+
,$$$$$bmmd$$$P^'
|
112
|
+
.d$$$$$$$$$$P'
|
113
|
+
$$^' `"^$$$' ____ _ _ _ _
|
114
|
+
$: ,$$: / ___|(_) __| | ___| | _(_) __ _
|
115
|
+
`b :$$ \___ \| |/ _` |/ _ \ |/ / |/ _` |
|
116
|
+
$$: ___) | | (_| | __/ <| | (_| |
|
117
|
+
$$ |____/|_|\__,_|\___|_|\_\_|\__, |
|
118
|
+
.d$$ |_|
|
119
|
+
}
|
120
|
+
end
|
121
|
+
|
122
|
+
def handle_signal(sig)
|
123
|
+
Sidekiq.logger.debug "Got #{sig} signal"
|
124
|
+
case sig
|
125
|
+
when 'INT'
|
126
|
+
# Handle Ctrl-C in JRuby like MRI
|
127
|
+
# http://jira.codehaus.org/browse/JRUBY-4637
|
128
|
+
raise Interrupt
|
129
|
+
when 'TERM'
|
130
|
+
# Heroku sends TERM and then waits 10 seconds for process to exit.
|
131
|
+
raise Interrupt
|
132
|
+
when 'USR1'
|
133
|
+
Sidekiq.logger.info "Received USR1, no longer accepting new work"
|
134
|
+
launcher.quiet
|
135
|
+
when 'USR2'
|
136
|
+
if Sidekiq.options[:logfile]
|
137
|
+
Sidekiq.logger.info "Received USR2, reopening log file"
|
138
|
+
Sidekiq::Logging.reopen_logs
|
139
|
+
end
|
140
|
+
when 'TTIN'
|
141
|
+
Thread.list.each do |thread|
|
142
|
+
Sidekiq.logger.warn "Thread TID-#{thread.object_id.to_s(36)} #{thread['label']}"
|
143
|
+
if thread.backtrace
|
144
|
+
Sidekiq.logger.warn thread.backtrace.join("\n")
|
145
|
+
else
|
146
|
+
Sidekiq.logger.warn "<no backtrace available>"
|
147
|
+
end
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
private
|
153
|
+
|
154
|
+
def print_banner
|
155
|
+
# Print logo and banner for development
|
156
|
+
if environment == 'development' && $stdout.tty?
|
157
|
+
puts "\e[#{31}m"
|
158
|
+
puts Sidekiq::CLI.banner
|
159
|
+
puts "\e[0m"
|
160
|
+
end
|
161
|
+
end
|
162
|
+
|
163
|
+
def daemonize
|
164
|
+
return unless options[:daemon]
|
165
|
+
|
166
|
+
raise ArgumentError, "You really should set a logfile if you're going to daemonize" unless options[:logfile]
|
167
|
+
files_to_reopen = []
|
168
|
+
ObjectSpace.each_object(File) do |file|
|
169
|
+
files_to_reopen << file unless file.closed?
|
170
|
+
end
|
171
|
+
|
172
|
+
::Process.daemon(true, true)
|
173
|
+
|
174
|
+
files_to_reopen.each do |file|
|
175
|
+
begin
|
176
|
+
file.reopen file.path, "a+"
|
177
|
+
file.sync = true
|
178
|
+
rescue ::Exception
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
[$stdout, $stderr].each do |io|
|
183
|
+
File.open(options[:logfile], 'ab') do |f|
|
184
|
+
io.reopen(f)
|
185
|
+
end
|
186
|
+
io.sync = true
|
187
|
+
end
|
188
|
+
$stdin.reopen('/dev/null')
|
189
|
+
|
190
|
+
initialize_logger
|
191
|
+
end
|
192
|
+
|
193
|
+
def set_environment(cli_env)
|
194
|
+
@environment = cli_env || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
|
195
|
+
end
|
196
|
+
|
197
|
+
alias_method :die, :exit
|
198
|
+
alias_method :☠, :exit
|
199
|
+
|
200
|
+
def setup_options(args)
|
201
|
+
opts = parse_options(args)
|
202
|
+
set_environment opts[:environment]
|
203
|
+
|
204
|
+
cfile = opts[:config_file]
|
205
|
+
opts = parse_config(cfile).merge(opts) if cfile
|
206
|
+
|
207
|
+
opts[:strict] = true if opts[:strict].nil?
|
208
|
+
|
209
|
+
options.merge!(opts)
|
210
|
+
end
|
211
|
+
|
212
|
+
def options
|
213
|
+
Sidekiq.options
|
214
|
+
end
|
215
|
+
|
216
|
+
def boot_system
|
217
|
+
ENV['RACK_ENV'] = ENV['RAILS_ENV'] = environment
|
218
|
+
|
219
|
+
raise ArgumentError, "#{options[:require]} does not exist" unless File.exist?(options[:require])
|
220
|
+
|
221
|
+
if File.directory?(options[:require])
|
222
|
+
require 'rails'
|
223
|
+
if ::Rails::VERSION::MAJOR < 4
|
224
|
+
require 'sidekiq/rails'
|
225
|
+
require File.expand_path("#{options[:require]}/config/environment.rb")
|
226
|
+
::Rails.application.eager_load!
|
227
|
+
else
|
228
|
+
# Painful contortions, see 1791 for discussion
|
229
|
+
require File.expand_path("#{options[:require]}/config/application.rb")
|
230
|
+
::Rails::Application.initializer "sidekiq.eager_load" do
|
231
|
+
::Rails.application.config.eager_load = true
|
232
|
+
end
|
233
|
+
require 'sidekiq/rails'
|
234
|
+
require File.expand_path("#{options[:require]}/config/environment.rb")
|
235
|
+
end
|
236
|
+
options[:tag] ||= default_tag
|
237
|
+
else
|
238
|
+
require options[:require]
|
239
|
+
end
|
240
|
+
end
|
241
|
+
|
242
|
+
def default_tag
|
243
|
+
dir = ::Rails.root
|
244
|
+
name = File.basename(dir)
|
245
|
+
if name.to_i != 0 && prevdir = File.dirname(dir) # Capistrano release directory?
|
246
|
+
if File.basename(prevdir) == 'releases'
|
247
|
+
return File.basename(File.dirname(prevdir))
|
248
|
+
end
|
249
|
+
end
|
250
|
+
name
|
251
|
+
end
|
252
|
+
|
253
|
+
def validate!
|
254
|
+
options[:queues] << 'default' if options[:queues].empty?
|
255
|
+
|
256
|
+
if !File.exist?(options[:require]) ||
|
257
|
+
(File.directory?(options[:require]) && !File.exist?("#{options[:require]}/config/application.rb"))
|
258
|
+
logger.info "=================================================================="
|
259
|
+
logger.info " Please point sidekiq to a Rails 3/4 application or a Ruby file "
|
260
|
+
logger.info " to load your worker classes with -r [DIR|FILE]."
|
261
|
+
logger.info "=================================================================="
|
262
|
+
logger.info @parser
|
263
|
+
die(1)
|
264
|
+
end
|
265
|
+
|
266
|
+
[:concurrency, :timeout].each do |opt|
|
267
|
+
raise ArgumentError, "#{opt}: #{options[opt]} is not a valid value" if options.has_key?(opt) && options[opt].to_i <= 0
|
268
|
+
end
|
269
|
+
end
|
270
|
+
|
271
|
+
def parse_options(argv)
|
272
|
+
opts = {}
|
273
|
+
|
274
|
+
@parser = OptionParser.new do |o|
|
275
|
+
o.on '-c', '--concurrency INT', "processor threads to use" do |arg|
|
276
|
+
opts[:concurrency] = Integer(arg)
|
277
|
+
end
|
278
|
+
|
279
|
+
o.on '-d', '--daemon', "Daemonize process" do |arg|
|
280
|
+
opts[:daemon] = arg
|
281
|
+
end
|
282
|
+
|
283
|
+
o.on '-e', '--environment ENV', "Application environment" do |arg|
|
284
|
+
opts[:environment] = arg
|
285
|
+
end
|
286
|
+
|
287
|
+
o.on '-g', '--tag TAG', "Process tag for procline" do |arg|
|
288
|
+
opts[:tag] = arg
|
289
|
+
end
|
290
|
+
|
291
|
+
o.on '-i', '--index INT', "unique process index on this machine" do |arg|
|
292
|
+
opts[:index] = Integer(arg.match(/\d+/)[0])
|
293
|
+
end
|
294
|
+
|
295
|
+
o.on "-q", "--queue QUEUE[,WEIGHT]", "Queues to process with optional weights" do |arg|
|
296
|
+
queue, weight = arg.split(",")
|
297
|
+
parse_queue opts, queue, weight
|
298
|
+
end
|
299
|
+
|
300
|
+
o.on '-r', '--require [PATH|DIR]', "Location of Rails application with workers or file to require" do |arg|
|
301
|
+
opts[:require] = arg
|
302
|
+
end
|
303
|
+
|
304
|
+
o.on '-t', '--timeout NUM', "Shutdown timeout" do |arg|
|
305
|
+
opts[:timeout] = Integer(arg)
|
306
|
+
end
|
307
|
+
|
308
|
+
o.on "-v", "--verbose", "Print more verbose output" do |arg|
|
309
|
+
opts[:verbose] = arg
|
310
|
+
end
|
311
|
+
|
312
|
+
o.on '-C', '--config PATH', "path to YAML config file" do |arg|
|
313
|
+
opts[:config_file] = arg
|
314
|
+
end
|
315
|
+
|
316
|
+
o.on '-L', '--logfile PATH', "path to writable logfile" do |arg|
|
317
|
+
opts[:logfile] = arg
|
318
|
+
end
|
319
|
+
|
320
|
+
o.on '-P', '--pidfile PATH', "path to pidfile" do |arg|
|
321
|
+
opts[:pidfile] = arg
|
322
|
+
end
|
323
|
+
|
324
|
+
o.on '-V', '--version', "Print version and exit" do |arg|
|
325
|
+
puts "Sidekiq #{Sidekiq::VERSION}"
|
326
|
+
die(0)
|
327
|
+
end
|
328
|
+
end
|
329
|
+
|
330
|
+
@parser.banner = "sidekiq [options]"
|
331
|
+
@parser.on_tail "-h", "--help", "Show help" do
|
332
|
+
logger.info @parser
|
333
|
+
die 1
|
334
|
+
end
|
335
|
+
@parser.parse!(argv)
|
336
|
+
|
337
|
+
%w[config/sidekiq.yml config/sidekiq.yml.erb].each do |filename|
|
338
|
+
opts[:config_file] ||= filename if File.exist?(filename)
|
339
|
+
end
|
340
|
+
|
341
|
+
opts
|
342
|
+
end
|
343
|
+
|
344
|
+
def initialize_logger
|
345
|
+
Sidekiq::Logging.initialize_logger(options[:logfile]) if options[:logfile]
|
346
|
+
|
347
|
+
Sidekiq.logger.level = ::Logger::DEBUG if options[:verbose]
|
348
|
+
end
|
349
|
+
|
350
|
+
def write_pid
|
351
|
+
if path = options[:pidfile]
|
352
|
+
pidfile = File.expand_path(path)
|
353
|
+
File.open(pidfile, 'w') do |f|
|
354
|
+
f.puts ::Process.pid
|
355
|
+
end
|
356
|
+
end
|
357
|
+
end
|
358
|
+
|
359
|
+
def parse_config(cfile)
|
360
|
+
opts = {}
|
361
|
+
if File.exist?(cfile)
|
362
|
+
opts = YAML.load(ERB.new(IO.read(cfile)).result) || opts
|
363
|
+
opts = opts.merge(opts.delete(environment) || {})
|
364
|
+
parse_queues(opts, opts.delete(:queues) || [])
|
365
|
+
else
|
366
|
+
# allow a non-existent config file so Sidekiq
|
367
|
+
# can be deployed by cap with just the defaults.
|
368
|
+
end
|
369
|
+
ns = opts.delete(:namespace)
|
370
|
+
if ns
|
371
|
+
# logger hasn't been initialized yet, puts is all we have.
|
372
|
+
puts("namespace should be set in your ruby initializer, is ignored in config file")
|
373
|
+
puts("config.redis = { :url => ..., :namespace => '#{ns}' }")
|
374
|
+
end
|
375
|
+
opts
|
376
|
+
end
|
377
|
+
|
378
|
+
def parse_queues(opts, queues_and_weights)
|
379
|
+
queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
|
380
|
+
end
|
381
|
+
|
382
|
+
def parse_queue(opts, q, weight=nil)
|
383
|
+
[weight.to_i, 1].max.times do
|
384
|
+
(opts[:queues] ||= []) << q
|
385
|
+
end
|
386
|
+
opts[:strict] = false if weight.to_i > 0
|
387
|
+
end
|
388
|
+
end
|
389
|
+
end
|
@@ -0,0 +1,260 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'securerandom'
|
3
|
+
require 'sidekiq/middleware/chain'
|
4
|
+
|
5
|
+
module Sidekiq
|
6
|
+
class Client
|
7
|
+
|
8
|
+
##
|
9
|
+
# Define client-side middleware:
|
10
|
+
#
|
11
|
+
# client = Sidekiq::Client.new
|
12
|
+
# client.middleware do |chain|
|
13
|
+
# chain.use MyClientMiddleware
|
14
|
+
# end
|
15
|
+
# client.push('class' => 'SomeWorker', 'args' => [1,2,3])
|
16
|
+
#
|
17
|
+
# All client instances default to the globally-defined
|
18
|
+
# Sidekiq.client_middleware but you can change as necessary.
|
19
|
+
#
|
20
|
+
def middleware(&block)
|
21
|
+
@chain ||= Sidekiq.client_middleware
|
22
|
+
if block_given?
|
23
|
+
@chain = @chain.dup
|
24
|
+
yield @chain
|
25
|
+
end
|
26
|
+
@chain
|
27
|
+
end
|
28
|
+
|
29
|
+
attr_accessor :redis_pool
|
30
|
+
|
31
|
+
# Sidekiq::Client normally uses the default Redis pool but you may
|
32
|
+
# pass a custom ConnectionPool if you want to shard your
|
33
|
+
# Sidekiq jobs across several Redis instances (for scalability
|
34
|
+
# reasons, e.g.)
|
35
|
+
#
|
36
|
+
# Sidekiq::Client.new(ConnectionPool.new { Redis.new })
|
37
|
+
#
|
38
|
+
# Generally this is only needed for very large Sidekiq installs processing
|
39
|
+
# thousands of jobs per second. I don't recommend sharding unless you
|
40
|
+
# cannot scale any other way (e.g. splitting your app into smaller apps).
|
41
|
+
def initialize(redis_pool=nil)
|
42
|
+
@redis_pool = redis_pool || Thread.current[:sidekiq_via_pool] || Sidekiq.redis_pool
|
43
|
+
end
|
44
|
+
|
45
|
+
##
|
46
|
+
# The main method used to push a job to Redis. Accepts a number of options:
|
47
|
+
#
|
48
|
+
# queue - the named queue to use, default 'default'
|
49
|
+
# class - the worker class to call, required
|
50
|
+
# args - an array of simple arguments to the perform method, must be JSON-serializable
|
51
|
+
# retry - whether to retry this job if it fails, default true or an integer number of retries
|
52
|
+
# backtrace - whether to save any error backtrace, default false
|
53
|
+
#
|
54
|
+
# All options must be strings, not symbols. NB: because we are serializing to JSON, all
|
55
|
+
# symbols in 'args' will be converted to strings. Note that +backtrace: true+ can take quite a bit of
|
56
|
+
# space in Redis; a large volume of failing jobs can start Redis swapping if you aren't careful.
|
57
|
+
#
|
58
|
+
# Returns a unique Job ID. If middleware stops the job, nil will be returned instead.
|
59
|
+
#
|
60
|
+
# Example:
|
61
|
+
# push('queue' => 'my_queue', 'class' => MyWorker, 'args' => ['foo', 1, :bat => 'bar'])
|
62
|
+
#
|
63
|
+
def push(item)
|
64
|
+
normed = normalize_item(item)
|
65
|
+
payload = process_single(item['class'], normed)
|
66
|
+
|
67
|
+
if payload
|
68
|
+
raw_push([payload])
|
69
|
+
payload['jid']
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
##
|
74
|
+
# Push a large number of jobs to Redis. In practice this method is only
|
75
|
+
# useful if you are pushing thousands of jobs or more. This method
|
76
|
+
# cuts out the redis network round trip latency.
|
77
|
+
#
|
78
|
+
# Takes the same arguments as #push except that args is expected to be
|
79
|
+
# an Array of Arrays. All other keys are duplicated for each job. Each job
|
80
|
+
# is run through the client middleware pipeline and each job gets its own Job ID
|
81
|
+
# as normal.
|
82
|
+
#
|
83
|
+
# Returns an array of the of pushed jobs' jids. The number of jobs pushed can be less
|
84
|
+
# than the number given if the middleware stopped processing for one or more jobs.
|
85
|
+
def push_bulk(items)
|
86
|
+
arg = items['args'].first
|
87
|
+
raise ArgumentError, "Bulk arguments must be an Array of Arrays: [[1], [2]]" if !arg.is_a?(Array)
|
88
|
+
|
89
|
+
normed = normalize_item(items)
|
90
|
+
payloads = items['args'].map do |args|
|
91
|
+
copy = normed.merge('args' => args, 'jid' => SecureRandom.hex(12), 'enqueued_at' => Time.now.to_f)
|
92
|
+
result = process_single(items['class'], copy)
|
93
|
+
result ? result : nil
|
94
|
+
end.compact
|
95
|
+
|
96
|
+
raw_push(payloads) if !payloads.empty?
|
97
|
+
payloads.collect { |payload| payload['jid'] }
|
98
|
+
end
|
99
|
+
|
100
|
+
# Allows sharding of jobs across any number of Redis instances. All jobs
|
101
|
+
# defined within the block will use the given Redis connection pool.
|
102
|
+
#
|
103
|
+
# pool = ConnectionPool.new { Redis.new }
|
104
|
+
# Sidekiq::Client.via(pool) do
|
105
|
+
# SomeWorker.perform_async(1,2,3)
|
106
|
+
# SomeOtherWorker.perform_async(1,2,3)
|
107
|
+
# end
|
108
|
+
#
|
109
|
+
# Generally this is only needed for very large Sidekiq installs processing
|
110
|
+
# thousands of jobs per second. I do not recommend sharding unless
|
111
|
+
# you cannot scale any other way (e.g. splitting your app into smaller apps).
|
112
|
+
def self.via(pool)
|
113
|
+
raise ArgumentError, "No pool given" if pool.nil?
|
114
|
+
current_sidekiq_pool = Thread.current[:sidekiq_via_pool]
|
115
|
+
raise RuntimeError, "Sidekiq::Client.via is not re-entrant" if current_sidekiq_pool && current_sidekiq_pool != pool
|
116
|
+
Thread.current[:sidekiq_via_pool] = pool
|
117
|
+
yield
|
118
|
+
ensure
|
119
|
+
Thread.current[:sidekiq_via_pool] = nil
|
120
|
+
end
|
121
|
+
|
122
|
+
class << self
|
123
|
+
|
124
|
+
def push(item)
|
125
|
+
new.push(item)
|
126
|
+
end
|
127
|
+
|
128
|
+
def push_bulk(items)
|
129
|
+
new.push_bulk(items)
|
130
|
+
end
|
131
|
+
|
132
|
+
# Resque compatibility helpers. Note all helpers
|
133
|
+
# should go through Worker#client_push.
|
134
|
+
#
|
135
|
+
# Example usage:
|
136
|
+
# Sidekiq::Client.enqueue(MyWorker, 'foo', 1, :bat => 'bar')
|
137
|
+
#
|
138
|
+
# Messages are enqueued to the 'default' queue.
|
139
|
+
#
|
140
|
+
def enqueue(klass, *args)
|
141
|
+
klass.client_push('class' => klass, 'args' => args)
|
142
|
+
end
|
143
|
+
|
144
|
+
# Example usage:
|
145
|
+
# Sidekiq::Client.enqueue_to(:queue_name, MyWorker, 'foo', 1, :bat => 'bar')
|
146
|
+
#
|
147
|
+
def enqueue_to(queue, klass, *args)
|
148
|
+
klass.client_push('queue' => queue, 'class' => klass, 'args' => args)
|
149
|
+
end
|
150
|
+
|
151
|
+
# Example usage:
|
152
|
+
# Sidekiq::Client.enqueue_to_in(:queue_name, 3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
|
153
|
+
#
|
154
|
+
def enqueue_to_in(queue, interval, klass, *args)
|
155
|
+
int = interval.to_f
|
156
|
+
now = Time.now.to_f
|
157
|
+
ts = (int < 1_000_000_000 ? now + int : int)
|
158
|
+
|
159
|
+
item = { 'class' => klass, 'args' => args, 'at' => ts, 'queue' => queue }
|
160
|
+
item.delete('at'.freeze) if ts <= now
|
161
|
+
|
162
|
+
klass.client_push(item)
|
163
|
+
end
|
164
|
+
|
165
|
+
# Example usage:
|
166
|
+
# Sidekiq::Client.enqueue_in(3.minutes, MyWorker, 'foo', 1, :bat => 'bar')
|
167
|
+
#
|
168
|
+
def enqueue_in(interval, klass, *args)
|
169
|
+
klass.perform_in(interval, *args)
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
private
|
174
|
+
|
175
|
+
def raw_push(payloads)
|
176
|
+
@redis_pool.with do |conn|
|
177
|
+
conn.multi do
|
178
|
+
atomic_push(conn, payloads)
|
179
|
+
end
|
180
|
+
end
|
181
|
+
true
|
182
|
+
end
|
183
|
+
|
184
|
+
def atomic_push(conn, payloads)
|
185
|
+
if payloads.first['at']
|
186
|
+
conn.zadd('schedule'.freeze, payloads.map do |hash|
|
187
|
+
at = hash.delete('at'.freeze).to_s
|
188
|
+
[at, Sidekiq.dump_json(hash)]
|
189
|
+
end)
|
190
|
+
else
|
191
|
+
q = payloads.first['queue']
|
192
|
+
now = Time.now.to_f
|
193
|
+
to_push = payloads.map do |entry|
|
194
|
+
entry['enqueued_at'.freeze] = now
|
195
|
+
Sidekiq.dump_json(entry)
|
196
|
+
end
|
197
|
+
conn.sadd('queues'.freeze, q)
|
198
|
+
conn.lpush("queue:#{q}", to_push)
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
def process_single(worker_class, item)
|
203
|
+
queue = item['queue']
|
204
|
+
|
205
|
+
middleware.invoke(worker_class, item, queue, @redis_pool) do
|
206
|
+
item
|
207
|
+
end
|
208
|
+
end
|
209
|
+
|
210
|
+
def normalize_item(item)
|
211
|
+
raise(ArgumentError, "Job must be a Hash with 'class' and 'args' keys: { 'class' => SomeWorker, 'args' => ['bob', 1, :foo => 'bar'] }") unless item.is_a?(Hash) && item.has_key?('class'.freeze) && item.has_key?('args'.freeze)
|
212
|
+
raise(ArgumentError, "Job args must be an Array") unless item['args'.freeze].is_a?(Array)
|
213
|
+
raise(ArgumentError, "Job class must be either a Class or String representation of the class name") unless item['class'.freeze].is_a?(Class) || item['class'.freeze].is_a?(String)
|
214
|
+
#raise(ArgumentError, "Arguments must be native JSON types, see https://github.com/mperham/sidekiq/wiki/Best-Practices") unless JSON.load(JSON.dump(item['args'])) == item['args']
|
215
|
+
|
216
|
+
unless item['retry'.freeze]
|
217
|
+
args_to_check = if item['class'.freeze].is_a?(Class) && item['class'.freeze].name.include?('ActiveJob'.freeze)
|
218
|
+
raise(ArgumentError, "Unexpected arguments (expected array with single element): #{item['args'.freeze].inspect}") unless item['args'.freeze].length == 1
|
219
|
+
raise(ArgumentError, "Expected item['args'][0] to be a Hash: #{item['args'.freeze][0].inspect}") unless item['args'.freeze][0].is_a?(Hash)
|
220
|
+
if item['args'.freeze][0]['job_class'.freeze].include?('ActionMailer'.freeze) && item['args'.freeze][0]['arguments'.freeze].length > 3
|
221
|
+
# for ActionMailer, the first argument in the name of the mailer class, the second argument is the mail method, and the 3rd argument is the delivery method ('deliver_now').
|
222
|
+
item['args'.freeze][0]['arguments'.freeze][3..-1]
|
223
|
+
else
|
224
|
+
# check if all active job arguments are integers
|
225
|
+
item['args'.freeze][0]['arguments'.freeze]
|
226
|
+
end
|
227
|
+
else
|
228
|
+
# just check that item['args'] are all integers
|
229
|
+
item['args'.freeze]
|
230
|
+
end
|
231
|
+
|
232
|
+
args_to_check.each do |arg|
|
233
|
+
next if arg.is_a?(Fixnum)
|
234
|
+
raise(ArgumentError, "Only Fixnums, Arrays of Fixnums, or Hashes with Fixnum values are permitted as arguments. Received: #{arg}") unless arg.is_a?(Array) || arg.is_a?(Hash)
|
235
|
+
(arg.is_a?(Hash) ? arg.values : arg).each do |sub_arg|
|
236
|
+
raise(ArgumentError, "Only Fixnum arguments are permitted. Received: #{sub_arg}") unless sub_arg.is_a?(Fixnum)
|
237
|
+
end
|
238
|
+
end
|
239
|
+
end
|
240
|
+
|
241
|
+
normalized_hash(item['class'.freeze])
|
242
|
+
.each{ |key, value| item[key] = value if item[key].nil? }
|
243
|
+
|
244
|
+
item['class'.freeze] = item['class'.freeze].to_s
|
245
|
+
item['queue'.freeze] = item['queue'.freeze].to_s
|
246
|
+
item['jid'.freeze] ||= SecureRandom.hex(12)
|
247
|
+
item['created_at'.freeze] ||= Time.now.to_f
|
248
|
+
item
|
249
|
+
end
|
250
|
+
|
251
|
+
def normalized_hash(item_class)
|
252
|
+
if item_class.is_a?(Class)
|
253
|
+
raise(ArgumentError, "Message must include a Sidekiq::Worker class, not class name: #{item_class.ancestors.inspect}") if !item_class.respond_to?('get_sidekiq_options'.freeze)
|
254
|
+
item_class.get_sidekiq_options
|
255
|
+
else
|
256
|
+
Sidekiq.default_worker_options
|
257
|
+
end
|
258
|
+
end
|
259
|
+
end
|
260
|
+
end
|