resque_admin-scheduler 1.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (33) hide show
  1. checksums.yaml +7 -0
  2. data/bin/migrate_to_timestamps_set.rb +16 -0
  3. data/exe/resque-scheduler +5 -0
  4. data/lib/resque-scheduler.rb +4 -0
  5. data/lib/resque/scheduler.rb +447 -0
  6. data/lib/resque/scheduler/cli.rb +147 -0
  7. data/lib/resque/scheduler/configuration.rb +73 -0
  8. data/lib/resque/scheduler/delaying_extensions.rb +324 -0
  9. data/lib/resque/scheduler/env.rb +89 -0
  10. data/lib/resque/scheduler/extension.rb +13 -0
  11. data/lib/resque/scheduler/failure_handler.rb +11 -0
  12. data/lib/resque/scheduler/lock.rb +4 -0
  13. data/lib/resque/scheduler/lock/base.rb +61 -0
  14. data/lib/resque/scheduler/lock/basic.rb +27 -0
  15. data/lib/resque/scheduler/lock/resilient.rb +78 -0
  16. data/lib/resque/scheduler/locking.rb +104 -0
  17. data/lib/resque/scheduler/logger_builder.rb +72 -0
  18. data/lib/resque/scheduler/plugin.rb +31 -0
  19. data/lib/resque/scheduler/scheduling_extensions.rb +141 -0
  20. data/lib/resque/scheduler/server.rb +268 -0
  21. data/lib/resque/scheduler/server/views/delayed.erb +63 -0
  22. data/lib/resque/scheduler/server/views/delayed_schedules.erb +20 -0
  23. data/lib/resque/scheduler/server/views/delayed_timestamp.erb +26 -0
  24. data/lib/resque/scheduler/server/views/requeue-params.erb +23 -0
  25. data/lib/resque/scheduler/server/views/scheduler.erb +58 -0
  26. data/lib/resque/scheduler/server/views/search.erb +72 -0
  27. data/lib/resque/scheduler/server/views/search_form.erb +8 -0
  28. data/lib/resque/scheduler/signal_handling.rb +40 -0
  29. data/lib/resque/scheduler/tasks.rb +25 -0
  30. data/lib/resque/scheduler/util.rb +39 -0
  31. data/lib/resque/scheduler/version.rb +7 -0
  32. data/tasks/resque_scheduler.rake +2 -0
  33. metadata +267 -0
@@ -0,0 +1,89 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ require 'English' # $PROCESS_ID
4
+
5
+ module Resque
6
+ module Scheduler
7
+ class Env
8
+ def initialize(options)
9
+ @options = options
10
+ @pidfile_path = nil
11
+ end
12
+
13
+ def setup
14
+ require 'resque'
15
+ require 'resque/scheduler'
16
+
17
+ setup_backgrounding
18
+ setup_pid_file
19
+ setup_scheduler_configuration
20
+ end
21
+
22
+ def cleanup
23
+ cleanup_pid_file
24
+ end
25
+
26
+ private
27
+
28
+ attr_reader :options, :pidfile_path
29
+
30
+ def setup_backgrounding
31
+ return unless options[:background]
32
+
33
+ # Need to set this here for conditional Process.daemon redirect of
34
+ # stderr/stdout to /dev/null
35
+ Resque::Scheduler.quiet = if options.key?(:quiet)
36
+ !!options[:quiet]
37
+ else
38
+ true
39
+ end
40
+
41
+ unless Process.respond_to?('daemon')
42
+ abort 'background option is set, which requires ruby >= 1.9'
43
+ end
44
+
45
+ Process.daemon(true, !Resque::Scheduler.quiet)
46
+ Resque.redis.client.reconnect
47
+ end
48
+
49
+ def setup_pid_file
50
+ return unless options[:pidfile]
51
+
52
+ @pidfile_path = File.expand_path(options[:pidfile])
53
+
54
+ File.open(pidfile_path, 'w') do |f|
55
+ f.puts $PROCESS_ID
56
+ end
57
+
58
+ at_exit { cleanup_pid_file }
59
+ end
60
+
61
+ def setup_scheduler_configuration
62
+ Resque::Scheduler.configure do |c|
63
+ c.app_name = options[:app_name] if options.key?(:app_name)
64
+
65
+ c.dynamic = !!options[:dynamic] if options.key?(:dynamic)
66
+
67
+ c.env = options[:env] if options.key(:env)
68
+
69
+ c.logfile = options[:logfile] if options.key?(:logfile)
70
+
71
+ c.logformat = options[:logformat] if options.key?(:logformat)
72
+
73
+ if psleep = options[:poll_sleep_amount] && !psleep.nil?
74
+ c.poll_sleep_amount = Float(psleep)
75
+ end
76
+
77
+ c.verbose = !!options[:verbose] if options.key?(:verbose)
78
+ end
79
+ end
80
+
81
+ def cleanup_pid_file
82
+ return unless pidfile_path
83
+
84
+ File.delete(pidfile_path) if File.exist?(pidfile_path)
85
+ @pidfile_path = nil
86
+ end
87
+ end
88
+ end
89
+ end
@@ -0,0 +1,13 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ require_relative 'scheduling_extensions'
4
+ require_relative 'delaying_extensions'
5
+
6
+ module Resque
7
+ module Scheduler
8
+ module Extension
9
+ include SchedulingExtensions
10
+ include DelayingExtensions
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,11 @@
1
+ module Resque
2
+ module Scheduler
3
+ class FailureHandler
4
+ def self.on_enqueue_failure(_, e)
5
+ Resque::Scheduler.log_error(
6
+ "#{e.class.name}: #{e.message} #{e.backtrace.inspect}"
7
+ )
8
+ end
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,4 @@
1
+ # vim:fileencoding=utf-8
2
+ %w(base basic resilient).each do |file|
3
+ require "resque/scheduler/lock/#{file}"
4
+ end
@@ -0,0 +1,61 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ module Resque
4
+ module Scheduler
5
+ module Lock
6
+ class Base
7
+ attr_reader :key
8
+ attr_accessor :timeout
9
+
10
+ def initialize(key, options = {})
11
+ @key = key
12
+
13
+ # 3 minute default timeout
14
+ @timeout = options[:timeout] || 60 * 3
15
+ end
16
+
17
+ # Attempts to acquire the lock. Returns true if successfully acquired.
18
+ def acquire!
19
+ raise NotImplementedError
20
+ end
21
+
22
+ def value
23
+ @value ||= [hostname, process_id].join(':')
24
+ end
25
+
26
+ # Returns true if you currently hold the lock.
27
+ def locked?
28
+ raise NotImplementedError
29
+ end
30
+
31
+ # Releases the lock.
32
+ def release!
33
+ Resque.redis.del(key) == 1
34
+ end
35
+
36
+ # Releases the lock iff we own it
37
+ def release
38
+ locked? && release!
39
+ end
40
+
41
+ private
42
+
43
+ # Extends the lock by `timeout` seconds.
44
+ def extend_lock!
45
+ Resque.redis.expire(key, timeout)
46
+ end
47
+
48
+ def hostname
49
+ local_hostname = Socket.gethostname
50
+ Socket.gethostbyname(local_hostname).first
51
+ rescue
52
+ local_hostname
53
+ end
54
+
55
+ def process_id
56
+ Process.pid
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,27 @@
1
+ # vim:fileencoding=utf-8
2
+ require_relative 'base'
3
+
4
+ module Resque
5
+ module Scheduler
6
+ module Lock
7
+ class Basic < Base
8
+ def acquire!
9
+ if Resque.redis.setnx(key, value)
10
+ extend_lock!
11
+ true
12
+ end
13
+ end
14
+
15
+ def locked?
16
+ if Resque.redis.get(key) == value
17
+ extend_lock!
18
+
19
+ return true if Resque.redis.get(key) == value
20
+ end
21
+
22
+ false
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
@@ -0,0 +1,78 @@
1
+ # vim:fileencoding=utf-8
2
+ require_relative 'base'
3
+
4
+ module Resque
5
+ module Scheduler
6
+ module Lock
7
+ class Resilient < Base
8
+ def acquire!
9
+ evalsha(:acquire, [key], [value]).to_i == 1
10
+ end
11
+
12
+ def locked?
13
+ evalsha(:locked, [key], [value]).to_i == 1
14
+ end
15
+
16
+ def timeout=(seconds)
17
+ if locked?
18
+ @timeout = seconds
19
+ @locked_sha = nil
20
+ @acquire_sha = nil
21
+ end
22
+ @timeout
23
+ end
24
+
25
+ private
26
+
27
+ def evalsha(script, keys, argv, refresh: false)
28
+ sha_method_name = "#{script}_sha"
29
+ Resque.redis.evalsha(
30
+ send(sha_method_name, refresh),
31
+ keys: keys,
32
+ argv: argv
33
+ )
34
+ rescue Redis::CommandError => e
35
+ if e.message =~ /NOSCRIPT/
36
+ refresh = true
37
+ retry
38
+ end
39
+ raise
40
+ end
41
+
42
+ def locked_sha(refresh = false)
43
+ @locked_sha = nil if refresh
44
+
45
+ @locked_sha ||=
46
+ Resque.redis.script(:load, <<-EOF.gsub(/^ {14}/, ''))
47
+ if redis.call('GET', KEYS[1]) == ARGV[1]
48
+ then
49
+ redis.call('EXPIRE', KEYS[1], #{timeout})
50
+
51
+ if redis.call('GET', KEYS[1]) == ARGV[1]
52
+ then
53
+ return 1
54
+ end
55
+ end
56
+
57
+ return 0
58
+ EOF
59
+ end
60
+
61
+ def acquire_sha(refresh = false)
62
+ @acquire_sha = nil if refresh
63
+
64
+ @acquire_sha ||=
65
+ Resque.redis.script(:load, <<-EOF.gsub(/^ {14}/, ''))
66
+ if redis.call('SETNX', KEYS[1], ARGV[1]) == 1
67
+ then
68
+ redis.call('EXPIRE', KEYS[1], #{timeout})
69
+ return 1
70
+ else
71
+ return 0
72
+ end
73
+ EOF
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,104 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ # ### Locking the scheduler process
4
+ #
5
+ # There are two places in resque-scheduler that need to be synchonized in order
6
+ # to be able to run redundant scheduler processes while ensuring jobs don't get
7
+ # queued multiple times when the master process changes.
8
+ #
9
+ # 1) Processing the delayed queues (jobs that are created from
10
+ # enqueue_at/enqueue_in, etc) 2) Processing the scheduled (cron-like) jobs from
11
+ # rufus-scheduler
12
+ #
13
+ # Protecting the delayed queues (#1) is relatively easy. A simple SETNX in
14
+ # redis would suffice. However, protecting the scheduled jobs is trickier
15
+ # because the clocks on machines could be slightly off or actual firing times
16
+ # could vary slightly due to load. If scheduler A's clock is slightly ahead of
17
+ # scheduler B's clock (since they are on different machines), when scheduler A
18
+ # dies, we need to ensure that scheduler B doesn't queue jobs that A already
19
+ # queued before it's death. (This all assumes that it is better to miss a few
20
+ # scheduled jobs than it is to run them multiple times for the same iteration.)
21
+ #
22
+ # To avoid queuing multiple jobs in the case of master fail-over, the master
23
+ # should remain the master as long as it can rather than a simple SETNX which
24
+ # would result in the master roll being passed around frequently.
25
+ #
26
+ # Locking Scheme: Each resque-scheduler process attempts to get the master lock
27
+ # via SETNX. Once obtained, it sets the expiration for 3 minutes
28
+ # (configurable). The master process continually updates the timeout on the
29
+ # lock key to be 3 minutes in the future in it's loop(s) (see `run`) and when
30
+ # jobs come out of rufus-scheduler (see `load_schedule_job`). That ensures
31
+ # that a minimum of 3 minutes must pass since the last queuing operation before
32
+ # a new master is chosen. If, for whatever reason, the master fails to update
33
+ # the expiration for 3 minutes, the key expires and the lock is up for grabs.
34
+ # If miraculously the original master comes back to life, it will realize it is
35
+ # no longer the master and stop processing jobs.
36
+ #
37
+ # The clocks on the scheduler machines can then be up to 3 minutes off from
38
+ # each other without the risk of queueing the same scheduled job twice during a
39
+ # master change. The catch is, in the event of a master change, no scheduled
40
+ # jobs will be queued during those 3 minutes. So, there is a trade off: the
41
+ # higher the timeout, the less likely scheduled jobs will be fired twice but
42
+ # greater chances of missing scheduled jobs. The lower the timeout, less
43
+ # likely jobs will be missed, greater the chances of jobs firing twice. If you
44
+ # don't care about jobs firing twice or are certain your machines' clocks are
45
+ # well in sync, a lower timeout is preferable. One thing to keep in mind: this
46
+ # only effects *scheduled* jobs - delayed jobs will never be lost or skipped
47
+ # since eventually a master will come online and it will process everything
48
+ # that is ready (no matter how old it is). Scheduled jobs work like cron - if
49
+ # you stop cron, no jobs fire while it's stopped and it doesn't fire jobs that
50
+ # were missed when it starts up again.
51
+
52
+ require_relative 'lock'
53
+
54
+ module Resque
55
+ module Scheduler
56
+ module Locking
57
+ def master_lock
58
+ @master_lock ||= build_master_lock
59
+ end
60
+
61
+ def supports_lua?
62
+ redis_master_version >= 2.5
63
+ end
64
+
65
+ def master?
66
+ master_lock.acquire! || master_lock.locked?
67
+ end
68
+
69
+ def release_master_lock!
70
+ warn "#{self}\#release_master_lock! is deprecated because it does " \
71
+ "not respect lock ownership. Use #{self}\#release_master_lock " \
72
+ "instead (at #{caller.first}"
73
+
74
+ master_lock.release!
75
+ end
76
+
77
+ def release_master_lock
78
+ master_lock.release
79
+ rescue Errno::EAGAIN, Errno::ECONNRESET, Redis::CannotConnectError
80
+ @master_lock = nil
81
+ end
82
+
83
+ private
84
+
85
+ def build_master_lock
86
+ if supports_lua?
87
+ Resque::Scheduler::Lock::Resilient.new(master_lock_key)
88
+ else
89
+ Resque::Scheduler::Lock::Basic.new(master_lock_key)
90
+ end
91
+ end
92
+
93
+ def master_lock_key
94
+ lock_prefix = ENV['RESQUE_SCHEDULER_MASTER_LOCK_PREFIX'] || ''
95
+ lock_prefix += ':' if lock_prefix != ''
96
+ "#{Resque.redis.namespace}:#{lock_prefix}resque_scheduler_master_lock"
97
+ end
98
+
99
+ def redis_master_version
100
+ Resque.redis.info['redis_version'].to_f
101
+ end
102
+ end
103
+ end
104
+ end
@@ -0,0 +1,72 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ require 'mono_logger'
4
+
5
+ module Resque
6
+ module Scheduler
7
+ # Just builds a logger, with specified verbosity and destination.
8
+ # The simplest example:
9
+ #
10
+ # Resque::Scheduler::LoggerBuilder.new.build
11
+ class LoggerBuilder
12
+ # Initializes new instance of the builder
13
+ #
14
+ # Pass :opts Hash with
15
+ # - :quiet if logger needs to be silent for all levels. Default - false
16
+ # - :verbose if there is a need in debug messages. Default - false
17
+ # - :log_dev to output logs into a desired file. Default - STDOUT
18
+ # - :format log format, either 'text' or 'json'. Default - 'text'
19
+ #
20
+ # Example:
21
+ #
22
+ # LoggerBuilder.new(
23
+ # :quiet => false, :verbose => true, :log_dev => 'log/scheduler.log'
24
+ # )
25
+ def initialize(opts = {})
26
+ @quiet = !!opts[:quiet]
27
+ @verbose = !!opts[:verbose]
28
+ @log_dev = opts[:log_dev] || $stdout
29
+ @format = opts[:format] || 'text'
30
+ end
31
+
32
+ # Returns an instance of MonoLogger
33
+ def build
34
+ logger = MonoLogger.new(@log_dev)
35
+ logger.level = level
36
+ logger.formatter = send(:"#{@format}_formatter")
37
+ logger
38
+ end
39
+
40
+ private
41
+
42
+ def level
43
+ if @verbose && !@quiet
44
+ MonoLogger::DEBUG
45
+ elsif !@quiet
46
+ MonoLogger::INFO
47
+ else
48
+ MonoLogger::FATAL
49
+ end
50
+ end
51
+
52
+ def text_formatter
53
+ proc do |severity, datetime, _progname, msg|
54
+ "resque-scheduler: [#{severity}] #{datetime.iso8601}: #{msg}\n"
55
+ end
56
+ end
57
+
58
+ def json_formatter
59
+ proc do |severity, datetime, progname, msg|
60
+ require 'json'
61
+ JSON.dump(
62
+ name: 'resque-scheduler',
63
+ progname: progname,
64
+ level: severity,
65
+ timestamp: datetime.iso8601,
66
+ msg: msg
67
+ ) + "\n"
68
+ end
69
+ end
70
+ end
71
+ end
72
+ end