resque_admin-scheduler 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/bin/migrate_to_timestamps_set.rb +16 -0
- data/exe/resque-scheduler +5 -0
- data/lib/resque-scheduler.rb +4 -0
- data/lib/resque/scheduler.rb +447 -0
- data/lib/resque/scheduler/cli.rb +147 -0
- data/lib/resque/scheduler/configuration.rb +73 -0
- data/lib/resque/scheduler/delaying_extensions.rb +324 -0
- data/lib/resque/scheduler/env.rb +89 -0
- data/lib/resque/scheduler/extension.rb +13 -0
- data/lib/resque/scheduler/failure_handler.rb +11 -0
- data/lib/resque/scheduler/lock.rb +4 -0
- data/lib/resque/scheduler/lock/base.rb +61 -0
- data/lib/resque/scheduler/lock/basic.rb +27 -0
- data/lib/resque/scheduler/lock/resilient.rb +78 -0
- data/lib/resque/scheduler/locking.rb +104 -0
- data/lib/resque/scheduler/logger_builder.rb +72 -0
- data/lib/resque/scheduler/plugin.rb +31 -0
- data/lib/resque/scheduler/scheduling_extensions.rb +141 -0
- data/lib/resque/scheduler/server.rb +268 -0
- data/lib/resque/scheduler/server/views/delayed.erb +63 -0
- data/lib/resque/scheduler/server/views/delayed_schedules.erb +20 -0
- data/lib/resque/scheduler/server/views/delayed_timestamp.erb +26 -0
- data/lib/resque/scheduler/server/views/requeue-params.erb +23 -0
- data/lib/resque/scheduler/server/views/scheduler.erb +58 -0
- data/lib/resque/scheduler/server/views/search.erb +72 -0
- data/lib/resque/scheduler/server/views/search_form.erb +8 -0
- data/lib/resque/scheduler/signal_handling.rb +40 -0
- data/lib/resque/scheduler/tasks.rb +25 -0
- data/lib/resque/scheduler/util.rb +39 -0
- data/lib/resque/scheduler/version.rb +7 -0
- data/tasks/resque_scheduler.rake +2 -0
- metadata +267 -0
@@ -0,0 +1,89 @@
|
|
1
|
+
# vim:fileencoding=utf-8
|
2
|
+
|
3
|
+
require 'English' # $PROCESS_ID
|
4
|
+
|
5
|
+
module Resque
|
6
|
+
module Scheduler
|
7
|
+
class Env
|
8
|
+
def initialize(options)
|
9
|
+
@options = options
|
10
|
+
@pidfile_path = nil
|
11
|
+
end
|
12
|
+
|
13
|
+
def setup
|
14
|
+
require 'resque'
|
15
|
+
require 'resque/scheduler'
|
16
|
+
|
17
|
+
setup_backgrounding
|
18
|
+
setup_pid_file
|
19
|
+
setup_scheduler_configuration
|
20
|
+
end
|
21
|
+
|
22
|
+
def cleanup
|
23
|
+
cleanup_pid_file
|
24
|
+
end
|
25
|
+
|
26
|
+
private
|
27
|
+
|
28
|
+
attr_reader :options, :pidfile_path
|
29
|
+
|
30
|
+
def setup_backgrounding
|
31
|
+
return unless options[:background]
|
32
|
+
|
33
|
+
# Need to set this here for conditional Process.daemon redirect of
|
34
|
+
# stderr/stdout to /dev/null
|
35
|
+
Resque::Scheduler.quiet = if options.key?(:quiet)
|
36
|
+
!!options[:quiet]
|
37
|
+
else
|
38
|
+
true
|
39
|
+
end
|
40
|
+
|
41
|
+
unless Process.respond_to?('daemon')
|
42
|
+
abort 'background option is set, which requires ruby >= 1.9'
|
43
|
+
end
|
44
|
+
|
45
|
+
Process.daemon(true, !Resque::Scheduler.quiet)
|
46
|
+
Resque.redis.client.reconnect
|
47
|
+
end
|
48
|
+
|
49
|
+
def setup_pid_file
|
50
|
+
return unless options[:pidfile]
|
51
|
+
|
52
|
+
@pidfile_path = File.expand_path(options[:pidfile])
|
53
|
+
|
54
|
+
File.open(pidfile_path, 'w') do |f|
|
55
|
+
f.puts $PROCESS_ID
|
56
|
+
end
|
57
|
+
|
58
|
+
at_exit { cleanup_pid_file }
|
59
|
+
end
|
60
|
+
|
61
|
+
def setup_scheduler_configuration
|
62
|
+
Resque::Scheduler.configure do |c|
|
63
|
+
c.app_name = options[:app_name] if options.key?(:app_name)
|
64
|
+
|
65
|
+
c.dynamic = !!options[:dynamic] if options.key?(:dynamic)
|
66
|
+
|
67
|
+
c.env = options[:env] if options.key(:env)
|
68
|
+
|
69
|
+
c.logfile = options[:logfile] if options.key?(:logfile)
|
70
|
+
|
71
|
+
c.logformat = options[:logformat] if options.key?(:logformat)
|
72
|
+
|
73
|
+
if psleep = options[:poll_sleep_amount] && !psleep.nil?
|
74
|
+
c.poll_sleep_amount = Float(psleep)
|
75
|
+
end
|
76
|
+
|
77
|
+
c.verbose = !!options[:verbose] if options.key?(:verbose)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def cleanup_pid_file
|
82
|
+
return unless pidfile_path
|
83
|
+
|
84
|
+
File.delete(pidfile_path) if File.exist?(pidfile_path)
|
85
|
+
@pidfile_path = nil
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
# vim:fileencoding=utf-8
|
2
|
+
|
3
|
+
module Resque
|
4
|
+
module Scheduler
|
5
|
+
module Lock
|
6
|
+
class Base
|
7
|
+
attr_reader :key
|
8
|
+
attr_accessor :timeout
|
9
|
+
|
10
|
+
def initialize(key, options = {})
|
11
|
+
@key = key
|
12
|
+
|
13
|
+
# 3 minute default timeout
|
14
|
+
@timeout = options[:timeout] || 60 * 3
|
15
|
+
end
|
16
|
+
|
17
|
+
# Attempts to acquire the lock. Returns true if successfully acquired.
|
18
|
+
def acquire!
|
19
|
+
raise NotImplementedError
|
20
|
+
end
|
21
|
+
|
22
|
+
def value
|
23
|
+
@value ||= [hostname, process_id].join(':')
|
24
|
+
end
|
25
|
+
|
26
|
+
# Returns true if you currently hold the lock.
|
27
|
+
def locked?
|
28
|
+
raise NotImplementedError
|
29
|
+
end
|
30
|
+
|
31
|
+
# Releases the lock.
|
32
|
+
def release!
|
33
|
+
Resque.redis.del(key) == 1
|
34
|
+
end
|
35
|
+
|
36
|
+
# Releases the lock iff we own it
|
37
|
+
def release
|
38
|
+
locked? && release!
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
# Extends the lock by `timeout` seconds.
|
44
|
+
def extend_lock!
|
45
|
+
Resque.redis.expire(key, timeout)
|
46
|
+
end
|
47
|
+
|
48
|
+
def hostname
|
49
|
+
local_hostname = Socket.gethostname
|
50
|
+
Socket.gethostbyname(local_hostname).first
|
51
|
+
rescue
|
52
|
+
local_hostname
|
53
|
+
end
|
54
|
+
|
55
|
+
def process_id
|
56
|
+
Process.pid
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# vim:fileencoding=utf-8
|
2
|
+
require_relative 'base'
|
3
|
+
|
4
|
+
module Resque
|
5
|
+
module Scheduler
|
6
|
+
module Lock
|
7
|
+
class Basic < Base
|
8
|
+
def acquire!
|
9
|
+
if Resque.redis.setnx(key, value)
|
10
|
+
extend_lock!
|
11
|
+
true
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def locked?
|
16
|
+
if Resque.redis.get(key) == value
|
17
|
+
extend_lock!
|
18
|
+
|
19
|
+
return true if Resque.redis.get(key) == value
|
20
|
+
end
|
21
|
+
|
22
|
+
false
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,78 @@
|
|
1
|
+
# vim:fileencoding=utf-8
|
2
|
+
require_relative 'base'
|
3
|
+
|
4
|
+
module Resque
|
5
|
+
module Scheduler
|
6
|
+
module Lock
|
7
|
+
class Resilient < Base
|
8
|
+
def acquire!
|
9
|
+
evalsha(:acquire, [key], [value]).to_i == 1
|
10
|
+
end
|
11
|
+
|
12
|
+
def locked?
|
13
|
+
evalsha(:locked, [key], [value]).to_i == 1
|
14
|
+
end
|
15
|
+
|
16
|
+
def timeout=(seconds)
|
17
|
+
if locked?
|
18
|
+
@timeout = seconds
|
19
|
+
@locked_sha = nil
|
20
|
+
@acquire_sha = nil
|
21
|
+
end
|
22
|
+
@timeout
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def evalsha(script, keys, argv, refresh: false)
|
28
|
+
sha_method_name = "#{script}_sha"
|
29
|
+
Resque.redis.evalsha(
|
30
|
+
send(sha_method_name, refresh),
|
31
|
+
keys: keys,
|
32
|
+
argv: argv
|
33
|
+
)
|
34
|
+
rescue Redis::CommandError => e
|
35
|
+
if e.message =~ /NOSCRIPT/
|
36
|
+
refresh = true
|
37
|
+
retry
|
38
|
+
end
|
39
|
+
raise
|
40
|
+
end
|
41
|
+
|
42
|
+
def locked_sha(refresh = false)
|
43
|
+
@locked_sha = nil if refresh
|
44
|
+
|
45
|
+
@locked_sha ||=
|
46
|
+
Resque.redis.script(:load, <<-EOF.gsub(/^ {14}/, ''))
|
47
|
+
if redis.call('GET', KEYS[1]) == ARGV[1]
|
48
|
+
then
|
49
|
+
redis.call('EXPIRE', KEYS[1], #{timeout})
|
50
|
+
|
51
|
+
if redis.call('GET', KEYS[1]) == ARGV[1]
|
52
|
+
then
|
53
|
+
return 1
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
return 0
|
58
|
+
EOF
|
59
|
+
end
|
60
|
+
|
61
|
+
def acquire_sha(refresh = false)
|
62
|
+
@acquire_sha = nil if refresh
|
63
|
+
|
64
|
+
@acquire_sha ||=
|
65
|
+
Resque.redis.script(:load, <<-EOF.gsub(/^ {14}/, ''))
|
66
|
+
if redis.call('SETNX', KEYS[1], ARGV[1]) == 1
|
67
|
+
then
|
68
|
+
redis.call('EXPIRE', KEYS[1], #{timeout})
|
69
|
+
return 1
|
70
|
+
else
|
71
|
+
return 0
|
72
|
+
end
|
73
|
+
EOF
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
@@ -0,0 +1,104 @@
|
|
1
|
+
# vim:fileencoding=utf-8
|
2
|
+
|
3
|
+
# ### Locking the scheduler process
|
4
|
+
#
|
5
|
+
# There are two places in resque-scheduler that need to be synchonized in order
|
6
|
+
# to be able to run redundant scheduler processes while ensuring jobs don't get
|
7
|
+
# queued multiple times when the master process changes.
|
8
|
+
#
|
9
|
+
# 1) Processing the delayed queues (jobs that are created from
|
10
|
+
# enqueue_at/enqueue_in, etc) 2) Processing the scheduled (cron-like) jobs from
|
11
|
+
# rufus-scheduler
|
12
|
+
#
|
13
|
+
# Protecting the delayed queues (#1) is relatively easy. A simple SETNX in
|
14
|
+
# redis would suffice. However, protecting the scheduled jobs is trickier
|
15
|
+
# because the clocks on machines could be slightly off or actual firing times
|
16
|
+
# could vary slightly due to load. If scheduler A's clock is slightly ahead of
|
17
|
+
# scheduler B's clock (since they are on different machines), when scheduler A
|
18
|
+
# dies, we need to ensure that scheduler B doesn't queue jobs that A already
|
19
|
+
# queued before it's death. (This all assumes that it is better to miss a few
|
20
|
+
# scheduled jobs than it is to run them multiple times for the same iteration.)
|
21
|
+
#
|
22
|
+
# To avoid queuing multiple jobs in the case of master fail-over, the master
|
23
|
+
# should remain the master as long as it can rather than a simple SETNX which
|
24
|
+
# would result in the master roll being passed around frequently.
|
25
|
+
#
|
26
|
+
# Locking Scheme: Each resque-scheduler process attempts to get the master lock
|
27
|
+
# via SETNX. Once obtained, it sets the expiration for 3 minutes
|
28
|
+
# (configurable). The master process continually updates the timeout on the
|
29
|
+
# lock key to be 3 minutes in the future in it's loop(s) (see `run`) and when
|
30
|
+
# jobs come out of rufus-scheduler (see `load_schedule_job`). That ensures
|
31
|
+
# that a minimum of 3 minutes must pass since the last queuing operation before
|
32
|
+
# a new master is chosen. If, for whatever reason, the master fails to update
|
33
|
+
# the expiration for 3 minutes, the key expires and the lock is up for grabs.
|
34
|
+
# If miraculously the original master comes back to life, it will realize it is
|
35
|
+
# no longer the master and stop processing jobs.
|
36
|
+
#
|
37
|
+
# The clocks on the scheduler machines can then be up to 3 minutes off from
|
38
|
+
# each other without the risk of queueing the same scheduled job twice during a
|
39
|
+
# master change. The catch is, in the event of a master change, no scheduled
|
40
|
+
# jobs will be queued during those 3 minutes. So, there is a trade off: the
|
41
|
+
# higher the timeout, the less likely scheduled jobs will be fired twice but
|
42
|
+
# greater chances of missing scheduled jobs. The lower the timeout, less
|
43
|
+
# likely jobs will be missed, greater the chances of jobs firing twice. If you
|
44
|
+
# don't care about jobs firing twice or are certain your machines' clocks are
|
45
|
+
# well in sync, a lower timeout is preferable. One thing to keep in mind: this
|
46
|
+
# only effects *scheduled* jobs - delayed jobs will never be lost or skipped
|
47
|
+
# since eventually a master will come online and it will process everything
|
48
|
+
# that is ready (no matter how old it is). Scheduled jobs work like cron - if
|
49
|
+
# you stop cron, no jobs fire while it's stopped and it doesn't fire jobs that
|
50
|
+
# were missed when it starts up again.
|
51
|
+
|
52
|
+
require_relative 'lock'
|
53
|
+
|
54
|
+
module Resque
|
55
|
+
module Scheduler
|
56
|
+
module Locking
|
57
|
+
def master_lock
|
58
|
+
@master_lock ||= build_master_lock
|
59
|
+
end
|
60
|
+
|
61
|
+
def supports_lua?
|
62
|
+
redis_master_version >= 2.5
|
63
|
+
end
|
64
|
+
|
65
|
+
def master?
|
66
|
+
master_lock.acquire! || master_lock.locked?
|
67
|
+
end
|
68
|
+
|
69
|
+
def release_master_lock!
|
70
|
+
warn "#{self}\#release_master_lock! is deprecated because it does " \
|
71
|
+
"not respect lock ownership. Use #{self}\#release_master_lock " \
|
72
|
+
"instead (at #{caller.first}"
|
73
|
+
|
74
|
+
master_lock.release!
|
75
|
+
end
|
76
|
+
|
77
|
+
def release_master_lock
|
78
|
+
master_lock.release
|
79
|
+
rescue Errno::EAGAIN, Errno::ECONNRESET, Redis::CannotConnectError
|
80
|
+
@master_lock = nil
|
81
|
+
end
|
82
|
+
|
83
|
+
private
|
84
|
+
|
85
|
+
def build_master_lock
|
86
|
+
if supports_lua?
|
87
|
+
Resque::Scheduler::Lock::Resilient.new(master_lock_key)
|
88
|
+
else
|
89
|
+
Resque::Scheduler::Lock::Basic.new(master_lock_key)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def master_lock_key
|
94
|
+
lock_prefix = ENV['RESQUE_SCHEDULER_MASTER_LOCK_PREFIX'] || ''
|
95
|
+
lock_prefix += ':' if lock_prefix != ''
|
96
|
+
"#{Resque.redis.namespace}:#{lock_prefix}resque_scheduler_master_lock"
|
97
|
+
end
|
98
|
+
|
99
|
+
def redis_master_version
|
100
|
+
Resque.redis.info['redis_version'].to_f
|
101
|
+
end
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
# vim:fileencoding=utf-8
|
2
|
+
|
3
|
+
require 'mono_logger'
|
4
|
+
|
5
|
+
module Resque
|
6
|
+
module Scheduler
|
7
|
+
# Just builds a logger, with specified verbosity and destination.
|
8
|
+
# The simplest example:
|
9
|
+
#
|
10
|
+
# Resque::Scheduler::LoggerBuilder.new.build
|
11
|
+
class LoggerBuilder
|
12
|
+
# Initializes new instance of the builder
|
13
|
+
#
|
14
|
+
# Pass :opts Hash with
|
15
|
+
# - :quiet if logger needs to be silent for all levels. Default - false
|
16
|
+
# - :verbose if there is a need in debug messages. Default - false
|
17
|
+
# - :log_dev to output logs into a desired file. Default - STDOUT
|
18
|
+
# - :format log format, either 'text' or 'json'. Default - 'text'
|
19
|
+
#
|
20
|
+
# Example:
|
21
|
+
#
|
22
|
+
# LoggerBuilder.new(
|
23
|
+
# :quiet => false, :verbose => true, :log_dev => 'log/scheduler.log'
|
24
|
+
# )
|
25
|
+
def initialize(opts = {})
|
26
|
+
@quiet = !!opts[:quiet]
|
27
|
+
@verbose = !!opts[:verbose]
|
28
|
+
@log_dev = opts[:log_dev] || $stdout
|
29
|
+
@format = opts[:format] || 'text'
|
30
|
+
end
|
31
|
+
|
32
|
+
# Returns an instance of MonoLogger
|
33
|
+
def build
|
34
|
+
logger = MonoLogger.new(@log_dev)
|
35
|
+
logger.level = level
|
36
|
+
logger.formatter = send(:"#{@format}_formatter")
|
37
|
+
logger
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
def level
|
43
|
+
if @verbose && !@quiet
|
44
|
+
MonoLogger::DEBUG
|
45
|
+
elsif !@quiet
|
46
|
+
MonoLogger::INFO
|
47
|
+
else
|
48
|
+
MonoLogger::FATAL
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def text_formatter
|
53
|
+
proc do |severity, datetime, _progname, msg|
|
54
|
+
"resque-scheduler: [#{severity}] #{datetime.iso8601}: #{msg}\n"
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def json_formatter
|
59
|
+
proc do |severity, datetime, progname, msg|
|
60
|
+
require 'json'
|
61
|
+
JSON.dump(
|
62
|
+
name: 'resque-scheduler',
|
63
|
+
progname: progname,
|
64
|
+
level: severity,
|
65
|
+
timestamp: datetime.iso8601,
|
66
|
+
msg: msg
|
67
|
+
) + "\n"
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|