resque_admin-scheduler 1.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. checksums.yaml +7 -0
  2. data/bin/migrate_to_timestamps_set.rb +16 -0
  3. data/exe/resque-scheduler +5 -0
  4. data/lib/resque-scheduler.rb +4 -0
  5. data/lib/resque/scheduler.rb +447 -0
  6. data/lib/resque/scheduler/cli.rb +147 -0
  7. data/lib/resque/scheduler/configuration.rb +73 -0
  8. data/lib/resque/scheduler/delaying_extensions.rb +324 -0
  9. data/lib/resque/scheduler/env.rb +89 -0
  10. data/lib/resque/scheduler/extension.rb +13 -0
  11. data/lib/resque/scheduler/failure_handler.rb +11 -0
  12. data/lib/resque/scheduler/lock.rb +4 -0
  13. data/lib/resque/scheduler/lock/base.rb +61 -0
  14. data/lib/resque/scheduler/lock/basic.rb +27 -0
  15. data/lib/resque/scheduler/lock/resilient.rb +78 -0
  16. data/lib/resque/scheduler/locking.rb +104 -0
  17. data/lib/resque/scheduler/logger_builder.rb +72 -0
  18. data/lib/resque/scheduler/plugin.rb +31 -0
  19. data/lib/resque/scheduler/scheduling_extensions.rb +141 -0
  20. data/lib/resque/scheduler/server.rb +268 -0
  21. data/lib/resque/scheduler/server/views/delayed.erb +63 -0
  22. data/lib/resque/scheduler/server/views/delayed_schedules.erb +20 -0
  23. data/lib/resque/scheduler/server/views/delayed_timestamp.erb +26 -0
  24. data/lib/resque/scheduler/server/views/requeue-params.erb +23 -0
  25. data/lib/resque/scheduler/server/views/scheduler.erb +58 -0
  26. data/lib/resque/scheduler/server/views/search.erb +72 -0
  27. data/lib/resque/scheduler/server/views/search_form.erb +8 -0
  28. data/lib/resque/scheduler/signal_handling.rb +40 -0
  29. data/lib/resque/scheduler/tasks.rb +25 -0
  30. data/lib/resque/scheduler/util.rb +39 -0
  31. data/lib/resque/scheduler/version.rb +7 -0
  32. data/tasks/resque_scheduler.rake +2 -0
  33. metadata +267 -0
@@ -0,0 +1,89 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ require 'English' # $PROCESS_ID
4
+
5
+ module Resque
6
+ module Scheduler
7
+ class Env
8
+ def initialize(options)
9
+ @options = options
10
+ @pidfile_path = nil
11
+ end
12
+
13
+ def setup
14
+ require 'resque'
15
+ require 'resque/scheduler'
16
+
17
+ setup_backgrounding
18
+ setup_pid_file
19
+ setup_scheduler_configuration
20
+ end
21
+
22
+ def cleanup
23
+ cleanup_pid_file
24
+ end
25
+
26
+ private
27
+
28
+ attr_reader :options, :pidfile_path
29
+
30
+ def setup_backgrounding
31
+ return unless options[:background]
32
+
33
+ # Need to set this here for conditional Process.daemon redirect of
34
+ # stderr/stdout to /dev/null
35
+ Resque::Scheduler.quiet = if options.key?(:quiet)
36
+ !!options[:quiet]
37
+ else
38
+ true
39
+ end
40
+
41
+ unless Process.respond_to?('daemon')
42
+ abort 'background option is set, which requires ruby >= 1.9'
43
+ end
44
+
45
+ Process.daemon(true, !Resque::Scheduler.quiet)
46
+ Resque.redis.client.reconnect
47
+ end
48
+
49
+ def setup_pid_file
50
+ return unless options[:pidfile]
51
+
52
+ @pidfile_path = File.expand_path(options[:pidfile])
53
+
54
+ File.open(pidfile_path, 'w') do |f|
55
+ f.puts $PROCESS_ID
56
+ end
57
+
58
+ at_exit { cleanup_pid_file }
59
+ end
60
+
61
+ def setup_scheduler_configuration
62
+ Resque::Scheduler.configure do |c|
63
+ c.app_name = options[:app_name] if options.key?(:app_name)
64
+
65
+ c.dynamic = !!options[:dynamic] if options.key?(:dynamic)
66
+
67
+ c.env = options[:env] if options.key(:env)
68
+
69
+ c.logfile = options[:logfile] if options.key?(:logfile)
70
+
71
+ c.logformat = options[:logformat] if options.key?(:logformat)
72
+
73
+ if psleep = options[:poll_sleep_amount] && !psleep.nil?
74
+ c.poll_sleep_amount = Float(psleep)
75
+ end
76
+
77
+ c.verbose = !!options[:verbose] if options.key?(:verbose)
78
+ end
79
+ end
80
+
81
+ def cleanup_pid_file
82
+ return unless pidfile_path
83
+
84
+ File.delete(pidfile_path) if File.exist?(pidfile_path)
85
+ @pidfile_path = nil
86
+ end
87
+ end
88
+ end
89
+ end
@@ -0,0 +1,13 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ require_relative 'scheduling_extensions'
4
+ require_relative 'delaying_extensions'
5
+
6
+ module Resque
7
+ module Scheduler
8
+ module Extension
9
+ include SchedulingExtensions
10
+ include DelayingExtensions
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,11 @@
1
+ module Resque
2
+ module Scheduler
3
+ class FailureHandler
4
+ def self.on_enqueue_failure(_, e)
5
+ Resque::Scheduler.log_error(
6
+ "#{e.class.name}: #{e.message} #{e.backtrace.inspect}"
7
+ )
8
+ end
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,4 @@
1
+ # vim:fileencoding=utf-8
2
+ %w(base basic resilient).each do |file|
3
+ require "resque/scheduler/lock/#{file}"
4
+ end
@@ -0,0 +1,61 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ module Resque
4
+ module Scheduler
5
+ module Lock
6
+ class Base
7
+ attr_reader :key
8
+ attr_accessor :timeout
9
+
10
+ def initialize(key, options = {})
11
+ @key = key
12
+
13
+ # 3 minute default timeout
14
+ @timeout = options[:timeout] || 60 * 3
15
+ end
16
+
17
+ # Attempts to acquire the lock. Returns true if successfully acquired.
18
+ def acquire!
19
+ raise NotImplementedError
20
+ end
21
+
22
+ def value
23
+ @value ||= [hostname, process_id].join(':')
24
+ end
25
+
26
+ # Returns true if you currently hold the lock.
27
+ def locked?
28
+ raise NotImplementedError
29
+ end
30
+
31
+ # Releases the lock.
32
+ def release!
33
+ Resque.redis.del(key) == 1
34
+ end
35
+
36
+ # Releases the lock iff we own it
37
+ def release
38
+ locked? && release!
39
+ end
40
+
41
+ private
42
+
43
+ # Extends the lock by `timeout` seconds.
44
+ def extend_lock!
45
+ Resque.redis.expire(key, timeout)
46
+ end
47
+
48
+ def hostname
49
+ local_hostname = Socket.gethostname
50
+ Socket.gethostbyname(local_hostname).first
51
+ rescue
52
+ local_hostname
53
+ end
54
+
55
+ def process_id
56
+ Process.pid
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,27 @@
1
+ # vim:fileencoding=utf-8
2
+ require_relative 'base'
3
+
4
+ module Resque
5
+ module Scheduler
6
+ module Lock
7
+ class Basic < Base
8
+ def acquire!
9
+ if Resque.redis.setnx(key, value)
10
+ extend_lock!
11
+ true
12
+ end
13
+ end
14
+
15
+ def locked?
16
+ if Resque.redis.get(key) == value
17
+ extend_lock!
18
+
19
+ return true if Resque.redis.get(key) == value
20
+ end
21
+
22
+ false
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,78 @@
1
+ # vim:fileencoding=utf-8
2
+ require_relative 'base'
3
+
4
+ module Resque
5
+ module Scheduler
6
+ module Lock
7
+ class Resilient < Base
8
+ def acquire!
9
+ evalsha(:acquire, [key], [value]).to_i == 1
10
+ end
11
+
12
+ def locked?
13
+ evalsha(:locked, [key], [value]).to_i == 1
14
+ end
15
+
16
+ def timeout=(seconds)
17
+ if locked?
18
+ @timeout = seconds
19
+ @locked_sha = nil
20
+ @acquire_sha = nil
21
+ end
22
+ @timeout
23
+ end
24
+
25
+ private
26
+
27
+ def evalsha(script, keys, argv, refresh: false)
28
+ sha_method_name = "#{script}_sha"
29
+ Resque.redis.evalsha(
30
+ send(sha_method_name, refresh),
31
+ keys: keys,
32
+ argv: argv
33
+ )
34
+ rescue Redis::CommandError => e
35
+ if e.message =~ /NOSCRIPT/
36
+ refresh = true
37
+ retry
38
+ end
39
+ raise
40
+ end
41
+
42
+ def locked_sha(refresh = false)
43
+ @locked_sha = nil if refresh
44
+
45
+ @locked_sha ||=
46
+ Resque.redis.script(:load, <<-EOF.gsub(/^ {14}/, ''))
47
+ if redis.call('GET', KEYS[1]) == ARGV[1]
48
+ then
49
+ redis.call('EXPIRE', KEYS[1], #{timeout})
50
+
51
+ if redis.call('GET', KEYS[1]) == ARGV[1]
52
+ then
53
+ return 1
54
+ end
55
+ end
56
+
57
+ return 0
58
+ EOF
59
+ end
60
+
61
+ def acquire_sha(refresh = false)
62
+ @acquire_sha = nil if refresh
63
+
64
+ @acquire_sha ||=
65
+ Resque.redis.script(:load, <<-EOF.gsub(/^ {14}/, ''))
66
+ if redis.call('SETNX', KEYS[1], ARGV[1]) == 1
67
+ then
68
+ redis.call('EXPIRE', KEYS[1], #{timeout})
69
+ return 1
70
+ else
71
+ return 0
72
+ end
73
+ EOF
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,104 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ # ### Locking the scheduler process
4
+ #
5
+ # There are two places in resque-scheduler that need to be synchonized in order
6
+ # to be able to run redundant scheduler processes while ensuring jobs don't get
7
+ # queued multiple times when the master process changes.
8
+ #
9
+ # 1) Processing the delayed queues (jobs that are created from
10
+ # enqueue_at/enqueue_in, etc) 2) Processing the scheduled (cron-like) jobs from
11
+ # rufus-scheduler
12
+ #
13
+ # Protecting the delayed queues (#1) is relatively easy. A simple SETNX in
14
+ # redis would suffice. However, protecting the scheduled jobs is trickier
15
+ # because the clocks on machines could be slightly off or actual firing times
16
+ # could vary slightly due to load. If scheduler A's clock is slightly ahead of
17
+ # scheduler B's clock (since they are on different machines), when scheduler A
18
+ # dies, we need to ensure that scheduler B doesn't queue jobs that A already
19
+ # queued before it's death. (This all assumes that it is better to miss a few
20
+ # scheduled jobs than it is to run them multiple times for the same iteration.)
21
+ #
22
+ # To avoid queuing multiple jobs in the case of master fail-over, the master
23
+ # should remain the master as long as it can rather than a simple SETNX which
24
+ # would result in the master roll being passed around frequently.
25
+ #
26
+ # Locking Scheme: Each resque-scheduler process attempts to get the master lock
27
+ # via SETNX. Once obtained, it sets the expiration for 3 minutes
28
+ # (configurable). The master process continually updates the timeout on the
29
+ # lock key to be 3 minutes in the future in it's loop(s) (see `run`) and when
30
+ # jobs come out of rufus-scheduler (see `load_schedule_job`). That ensures
31
+ # that a minimum of 3 minutes must pass since the last queuing operation before
32
+ # a new master is chosen. If, for whatever reason, the master fails to update
33
+ # the expiration for 3 minutes, the key expires and the lock is up for grabs.
34
+ # If miraculously the original master comes back to life, it will realize it is
35
+ # no longer the master and stop processing jobs.
36
+ #
37
+ # The clocks on the scheduler machines can then be up to 3 minutes off from
38
+ # each other without the risk of queueing the same scheduled job twice during a
39
+ # master change. The catch is, in the event of a master change, no scheduled
40
+ # jobs will be queued during those 3 minutes. So, there is a trade off: the
41
+ # higher the timeout, the less likely scheduled jobs will be fired twice but
42
+ # greater chances of missing scheduled jobs. The lower the timeout, less
43
+ # likely jobs will be missed, greater the chances of jobs firing twice. If you
44
+ # don't care about jobs firing twice or are certain your machines' clocks are
45
+ # well in sync, a lower timeout is preferable. One thing to keep in mind: this
46
+ # only effects *scheduled* jobs - delayed jobs will never be lost or skipped
47
+ # since eventually a master will come online and it will process everything
48
+ # that is ready (no matter how old it is). Scheduled jobs work like cron - if
49
+ # you stop cron, no jobs fire while it's stopped and it doesn't fire jobs that
50
+ # were missed when it starts up again.
51
+
52
+ require_relative 'lock'
53
+
54
+ module Resque
55
+ module Scheduler
56
+ module Locking
57
+ def master_lock
58
+ @master_lock ||= build_master_lock
59
+ end
60
+
61
+ def supports_lua?
62
+ redis_master_version >= 2.5
63
+ end
64
+
65
+ def master?
66
+ master_lock.acquire! || master_lock.locked?
67
+ end
68
+
69
+ def release_master_lock!
70
+ warn "#{self}\#release_master_lock! is deprecated because it does " \
71
+ "not respect lock ownership. Use #{self}\#release_master_lock " \
72
+ "instead (at #{caller.first}"
73
+
74
+ master_lock.release!
75
+ end
76
+
77
+ def release_master_lock
78
+ master_lock.release
79
+ rescue Errno::EAGAIN, Errno::ECONNRESET, Redis::CannotConnectError
80
+ @master_lock = nil
81
+ end
82
+
83
+ private
84
+
85
+ def build_master_lock
86
+ if supports_lua?
87
+ Resque::Scheduler::Lock::Resilient.new(master_lock_key)
88
+ else
89
+ Resque::Scheduler::Lock::Basic.new(master_lock_key)
90
+ end
91
+ end
92
+
93
+ def master_lock_key
94
+ lock_prefix = ENV['RESQUE_SCHEDULER_MASTER_LOCK_PREFIX'] || ''
95
+ lock_prefix += ':' if lock_prefix != ''
96
+ "#{Resque.redis.namespace}:#{lock_prefix}resque_scheduler_master_lock"
97
+ end
98
+
99
+ def redis_master_version
100
+ Resque.redis.info['redis_version'].to_f
101
+ end
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,72 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ require 'mono_logger'
4
+
5
+ module Resque
6
+ module Scheduler
7
+ # Just builds a logger, with specified verbosity and destination.
8
+ # The simplest example:
9
+ #
10
+ # Resque::Scheduler::LoggerBuilder.new.build
11
+ class LoggerBuilder
12
+ # Initializes new instance of the builder
13
+ #
14
+ # Pass :opts Hash with
15
+ # - :quiet if logger needs to be silent for all levels. Default - false
16
+ # - :verbose if there is a need in debug messages. Default - false
17
+ # - :log_dev to output logs into a desired file. Default - STDOUT
18
+ # - :format log format, either 'text' or 'json'. Default - 'text'
19
+ #
20
+ # Example:
21
+ #
22
+ # LoggerBuilder.new(
23
+ # :quiet => false, :verbose => true, :log_dev => 'log/scheduler.log'
24
+ # )
25
+ def initialize(opts = {})
26
+ @quiet = !!opts[:quiet]
27
+ @verbose = !!opts[:verbose]
28
+ @log_dev = opts[:log_dev] || $stdout
29
+ @format = opts[:format] || 'text'
30
+ end
31
+
32
+ # Returns an instance of MonoLogger
33
+ def build
34
+ logger = MonoLogger.new(@log_dev)
35
+ logger.level = level
36
+ logger.formatter = send(:"#{@format}_formatter")
37
+ logger
38
+ end
39
+
40
+ private
41
+
42
+ def level
43
+ if @verbose && !@quiet
44
+ MonoLogger::DEBUG
45
+ elsif !@quiet
46
+ MonoLogger::INFO
47
+ else
48
+ MonoLogger::FATAL
49
+ end
50
+ end
51
+
52
+ def text_formatter
53
+ proc do |severity, datetime, _progname, msg|
54
+ "resque-scheduler: [#{severity}] #{datetime.iso8601}: #{msg}\n"
55
+ end
56
+ end
57
+
58
+ def json_formatter
59
+ proc do |severity, datetime, progname, msg|
60
+ require 'json'
61
+ JSON.dump(
62
+ name: 'resque-scheduler',
63
+ progname: progname,
64
+ level: severity,
65
+ timestamp: datetime.iso8601,
66
+ msg: msg
67
+ ) + "\n"
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end