resque-scheduler 2.5.5 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of resque-scheduler might be problematic. Click here for more details.

Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +12 -6
  3. data/.rubocop.yml +18 -113
  4. data/.rubocop_todo.yml +29 -0
  5. data/.simplecov +3 -1
  6. data/.travis.yml +12 -4
  7. data/.vagrant-provision-as-vagrant.sh +15 -0
  8. data/.vagrant-provision.sh +23 -0
  9. data/.vagrant-skel/bash_profile +7 -0
  10. data/.vagrant-skel/bashrc +7 -0
  11. data/AUTHORS.md +5 -0
  12. data/Gemfile +1 -2
  13. data/HISTORY.md +18 -0
  14. data/README.md +39 -11
  15. data/ROADMAP.md +0 -6
  16. data/Rakefile +11 -19
  17. data/Vagrantfile +14 -0
  18. data/bin/resque-scheduler +2 -2
  19. data/examples/Rakefile +1 -1
  20. data/examples/config/initializers/resque-web.rb +2 -2
  21. data/examples/dynamic-scheduling/app/jobs/fix_schedules_job.rb +2 -4
  22. data/examples/dynamic-scheduling/app/jobs/send_email_job.rb +1 -1
  23. data/examples/dynamic-scheduling/app/models/user.rb +2 -2
  24. data/examples/dynamic-scheduling/lib/tasks/resque.rake +2 -2
  25. data/lib/resque-scheduler.rb +3 -1
  26. data/lib/resque/scheduler.rb +112 -168
  27. data/lib/resque/scheduler/cli.rb +144 -0
  28. data/lib/resque/scheduler/configuration.rb +73 -0
  29. data/lib/resque/scheduler/delaying_extensions.rb +278 -0
  30. data/lib/resque/scheduler/env.rb +61 -0
  31. data/lib/resque/scheduler/extension.rb +13 -0
  32. data/lib/resque/scheduler/lock.rb +2 -1
  33. data/lib/resque/scheduler/lock/base.rb +6 -2
  34. data/lib/resque/scheduler/lock/basic.rb +4 -5
  35. data/lib/resque/scheduler/lock/resilient.rb +30 -37
  36. data/lib/resque/scheduler/locking.rb +94 -0
  37. data/lib/resque/scheduler/logger_builder.rb +72 -0
  38. data/lib/resque/scheduler/plugin.rb +31 -0
  39. data/lib/resque/scheduler/scheduling_extensions.rb +150 -0
  40. data/lib/resque/scheduler/server.rb +246 -0
  41. data/lib/{resque_scheduler → resque/scheduler}/server/views/delayed.erb +2 -1
  42. data/lib/{resque_scheduler → resque/scheduler}/server/views/delayed_schedules.erb +0 -0
  43. data/lib/{resque_scheduler → resque/scheduler}/server/views/delayed_timestamp.erb +0 -0
  44. data/lib/{resque_scheduler → resque/scheduler}/server/views/requeue-params.erb +0 -0
  45. data/lib/{resque_scheduler → resque/scheduler}/server/views/scheduler.erb +16 -1
  46. data/lib/{resque_scheduler → resque/scheduler}/server/views/search.erb +2 -1
  47. data/lib/{resque_scheduler → resque/scheduler}/server/views/search_form.erb +0 -0
  48. data/lib/resque/scheduler/signal_handling.rb +40 -0
  49. data/lib/{resque_scheduler → resque/scheduler}/tasks.rb +3 -5
  50. data/lib/resque/scheduler/util.rb +41 -0
  51. data/lib/resque/scheduler/version.rb +7 -0
  52. data/resque-scheduler.gemspec +21 -19
  53. data/script/migrate_to_timestamps_set.rb +5 -3
  54. data/tasks/resque_scheduler.rake +1 -1
  55. data/test/cli_test.rb +26 -69
  56. data/test/delayed_queue_test.rb +262 -169
  57. data/test/env_test.rb +41 -0
  58. data/test/resque-web_test.rb +169 -48
  59. data/test/scheduler_args_test.rb +73 -41
  60. data/test/scheduler_hooks_test.rb +9 -8
  61. data/test/scheduler_locking_test.rb +55 -36
  62. data/test/scheduler_setup_test.rb +52 -15
  63. data/test/scheduler_task_test.rb +15 -10
  64. data/test/scheduler_test.rb +215 -114
  65. data/test/support/redis_instance.rb +32 -33
  66. data/test/test_helper.rb +33 -36
  67. data/test/util_test.rb +11 -0
  68. metadata +113 -57
  69. data/lib/resque/scheduler_locking.rb +0 -91
  70. data/lib/resque_scheduler.rb +0 -386
  71. data/lib/resque_scheduler/cli.rb +0 -160
  72. data/lib/resque_scheduler/logger_builder.rb +0 -72
  73. data/lib/resque_scheduler/plugin.rb +0 -28
  74. data/lib/resque_scheduler/server.rb +0 -183
  75. data/lib/resque_scheduler/util.rb +0 -34
  76. data/lib/resque_scheduler/version.rb +0 -5
  77. data/test/redis-test.conf +0 -108
@@ -0,0 +1,144 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ require 'optparse'
4
+
5
+ module Resque
6
+ module Scheduler
7
+ CLI_OPTIONS_ENV_MAPPING = {
8
+ app_name: 'APP_NAME',
9
+ background: 'BACKGROUND',
10
+ dynamic: 'DYNAMIC_SCHEDULE',
11
+ env: 'RAILS_ENV',
12
+ initializer_path: 'INITIALIZER_PATH',
13
+ logfile: 'LOGFILE',
14
+ logformat: 'LOGFORMAT',
15
+ quiet: 'QUIET',
16
+ pidfile: 'PIDFILE',
17
+ poll_sleep_amount: 'RESQUE_SCHEDULER_INTERVAL',
18
+ verbose: 'VERBOSE'
19
+ }
20
+
21
+ class Cli
22
+ BANNER = <<-EOF.gsub(/ {6}/, '')
23
+ Usage: resque-scheduler [options]
24
+
25
+ Runs a resque scheduler process directly (rather than via rake).
26
+
27
+ EOF
28
+ OPTIONS = [
29
+ {
30
+ args: ['-n', '--app-name [APP_NAME]',
31
+ 'Application name for procline'],
32
+ callback: ->(options) { ->(n) { options[:app_name] = n } }
33
+ },
34
+ {
35
+ args: ['-B', '--background', 'Run in the background [BACKGROUND]'],
36
+ callback: ->(options) { ->(b) { options[:background] = b } }
37
+ },
38
+ {
39
+ args: ['-D', '--dynamic-schedule',
40
+ 'Enable dynamic scheduling [DYNAMIC_SCHEDULE]'],
41
+ callback: ->(options) { ->(d) { options[:dynamic] = d } }
42
+ },
43
+ {
44
+ args: ['-E', '--environment [RAILS_ENV]', 'Environment name'],
45
+ callback: ->(options) { ->(e) { options[:env] = e } }
46
+ },
47
+ {
48
+ args: ['-I', '--initializer-path [INITIALIZER_PATH]',
49
+ 'Path to optional initializer ruby file'],
50
+ callback: ->(options) { ->(i) { options[:initializer_path] = i } }
51
+ },
52
+ {
53
+ args: ['-i', '--interval [RESQUE_SCHEDULER_INTERVAL]',
54
+ 'Interval for checking if a scheduled job must run'],
55
+ callback: ->(options) { ->(i) { options[:poll_sleep_amount] = i } }
56
+ },
57
+ {
58
+ args: ['-l', '--logfile [LOGFILE]', 'Log file name'],
59
+ callback: ->(options) { ->(l) { options[:logfile] = l } }
60
+ },
61
+ {
62
+ args: ['-F', '--logformat [LOGFORMAT]', 'Log output format'],
63
+ callback: ->(options) { ->(f) { options[:logformat] = f } }
64
+ },
65
+ {
66
+ args: ['-P', '--pidfile [PIDFILE]', 'PID file name'],
67
+ callback: ->(options) { ->(p) { options[:pidfile] = p } }
68
+ },
69
+ {
70
+ args: ['-q', '--quiet', 'Run with minimal output [QUIET]'],
71
+ callback: ->(options) { ->(q) { options[:quiet] = q } }
72
+ },
73
+ {
74
+ args: ['-v', '--verbose', 'Run with verbose output [VERBOSE]'],
75
+ callback: ->(options) { ->(v) { options[:verbose] = v } }
76
+ }
77
+ ].freeze
78
+
79
+ def self.run!(argv = ARGV, env = ENV)
80
+ new(argv, env).run!
81
+ end
82
+
83
+ def initialize(argv = ARGV, env = ENV)
84
+ @argv = argv
85
+ @env = env
86
+ end
87
+
88
+ def run!
89
+ pre_run
90
+ run_forever
91
+ end
92
+
93
+ def pre_run
94
+ parse_options
95
+ pre_setup
96
+ setup_env
97
+ end
98
+
99
+ def parse_options
100
+ option_parser.parse!(argv.dup)
101
+ end
102
+
103
+ def pre_setup
104
+ if options[:initializer_path]
105
+ load options[:initializer_path].to_s.strip
106
+ else
107
+ false
108
+ end
109
+ end
110
+
111
+ def setup_env
112
+ require_relative 'env'
113
+ runtime_env.setup
114
+ end
115
+
116
+ def run_forever
117
+ Resque::Scheduler.run
118
+ end
119
+
120
+ private
121
+
122
+ attr_reader :argv, :env
123
+
124
+ def runtime_env
125
+ Resque::Scheduler::Env.new(options)
126
+ end
127
+
128
+ def option_parser
129
+ OptionParser.new do |opts|
130
+ opts.banner = BANNER
131
+ OPTIONS.each do |opt|
132
+ opts.on(*opt[:args], &(opt[:callback].call(options)))
133
+ end
134
+ end
135
+ end
136
+
137
+ def options
138
+ @options ||= {}.tap do |o|
139
+ CLI_OPTIONS_ENV_MAPPING.map { |key, envvar| o[key] = env[envvar] }
140
+ end
141
+ end
142
+ end
143
+ end
144
+ end
@@ -0,0 +1,73 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ module Resque
4
+ module Scheduler
5
+ module Configuration
6
+ # Allows for block-style configuration
7
+ def configure
8
+ yield self
9
+ end
10
+
11
+ # Used in `#load_schedule_job`
12
+ attr_writer :env
13
+
14
+ def env
15
+ return @env if @env
16
+ @env ||= Rails.env if defined?(Rails)
17
+ @env ||= ENV['RAILS_ENV']
18
+ @env
19
+ end
20
+
21
+ # If true, logs more stuff...
22
+ attr_writer :verbose
23
+
24
+ def verbose
25
+ @verbose ||= !!ENV['VERBOSE']
26
+ end
27
+
28
+ # If set, produces no output
29
+ attr_writer :quiet
30
+
31
+ def quiet
32
+ @quiet ||= !!ENV['QUIET']
33
+ end
34
+
35
+ # If set, will write messages to the file
36
+ attr_writer :logfile
37
+
38
+ def logfile
39
+ @logfile ||= ENV['LOGFILE']
40
+ end
41
+
42
+ # Sets whether to log in 'text' or 'json'
43
+ attr_writer :logformat
44
+
45
+ def logformat
46
+ @logformat ||= ENV['LOGFORMAT']
47
+ end
48
+
49
+ # If set, will try to update the schedule in the loop
50
+ attr_writer :dynamic
51
+
52
+ def dynamic
53
+ @dynamic ||= !!ENV['DYNAMIC_SCHEDULE']
54
+ end
55
+
56
+ # If set, will append the app name to procline
57
+ attr_writer :app_name
58
+
59
+ def app_name
60
+ @app_name ||= ENV['APP_NAME']
61
+ end
62
+
63
+ # Amount of time in seconds to sleep between polls of the delayed
64
+ # queue. Defaults to 5
65
+ attr_writer :poll_sleep_amount
66
+
67
+ def poll_sleep_amount
68
+ @poll_sleep_amount ||=
69
+ Float(ENV.fetch('RESQUE_SCHEDULER_INTERVAL', '5'))
70
+ end
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,278 @@
1
+ # vim:fileencoding=utf-8
2
+ require 'resque'
3
+ require_relative 'plugin'
4
+ require_relative '../scheduler'
5
+
6
+ module Resque
7
+ module Scheduler
8
+ module DelayingExtensions
9
+ # This method is nearly identical to +enqueue+ only it also
10
+ # takes a timestamp which will be used to schedule the job
11
+ # for queueing. Until timestamp is in the past, the job will
12
+ # sit in the schedule list.
13
+ def enqueue_at(timestamp, klass, *args)
14
+ validate(klass)
15
+ enqueue_at_with_queue(
16
+ queue_from_class(klass), timestamp, klass, *args
17
+ )
18
+ end
19
+
20
+ # Identical to +enqueue_at+, except you can also specify
21
+ # a queue in which the job will be placed after the
22
+ # timestamp has passed. It respects Resque.inline option, by
23
+ # creating the job right away instead of adding to the queue.
24
+ def enqueue_at_with_queue(queue, timestamp, klass, *args)
25
+ return false unless plugin.run_before_schedule_hooks(klass, *args)
26
+
27
+ if Resque.inline? || timestamp.to_i < Time.now.to_i
28
+ # Just create the job and let resque perform it right away with
29
+ # inline. If the class is a custom job class, call self#scheduled
30
+ # on it. This allows you to do things like
31
+ # Resque.enqueue_at(timestamp, CustomJobClass, :opt1 => val1).
32
+ # Otherwise, pass off to Resque.
33
+ if klass.respond_to?(:scheduled)
34
+ klass.scheduled(queue, klass.to_s, *args)
35
+ else
36
+ Resque::Job.create(queue, klass, *args)
37
+ end
38
+ else
39
+ delayed_push(timestamp, job_to_hash_with_queue(queue, klass, args))
40
+ end
41
+
42
+ plugin.run_after_schedule_hooks(klass, *args)
43
+ end
44
+
45
+ # Identical to enqueue_at but takes number_of_seconds_from_now
46
+ # instead of a timestamp.
47
+ def enqueue_in(number_of_seconds_from_now, klass, *args)
48
+ enqueue_at(Time.now + number_of_seconds_from_now, klass, *args)
49
+ end
50
+
51
+ # Identical to +enqueue_in+, except you can also specify
52
+ # a queue in which the job will be placed after the
53
+ # number of seconds has passed.
54
+ def enqueue_in_with_queue(queue, number_of_seconds_from_now,
55
+ klass, *args)
56
+ enqueue_at_with_queue(queue, Time.now + number_of_seconds_from_now,
57
+ klass, *args)
58
+ end
59
+
60
+ # Used internally to stuff the item into the schedule sorted list.
61
+ # +timestamp+ can be either in seconds or a datetime object Insertion
62
+ # if O(log(n)). Returns true if it's the first job to be scheduled at
63
+ # that time, else false
64
+ def delayed_push(timestamp, item)
65
+ # First add this item to the list for this timestamp
66
+ redis.rpush("delayed:#{timestamp.to_i}", encode(item))
67
+
68
+ # Store the timestamps at with this item occurs
69
+ redis.sadd("timestamps:#{encode(item)}", "delayed:#{timestamp.to_i}")
70
+
71
+ # Now, add this timestamp to the zsets. The score and the value are
72
+ # the same since we'll be querying by timestamp, and we don't have
73
+ # anything else to store.
74
+ redis.zadd :delayed_queue_schedule, timestamp.to_i, timestamp.to_i
75
+ end
76
+
77
+ # Returns an array of timestamps based on start and count
78
+ def delayed_queue_peek(start, count)
79
+ result = redis.zrange(:delayed_queue_schedule, start,
80
+ start + count - 1)
81
+ Array(result).map(&:to_i)
82
+ end
83
+
84
+ # Returns the size of the delayed queue schedule
85
+ def delayed_queue_schedule_size
86
+ redis.zcard :delayed_queue_schedule
87
+ end
88
+
89
+ # Returns the number of jobs for a given timestamp in the delayed queue
90
+ # schedule
91
+ def delayed_timestamp_size(timestamp)
92
+ redis.llen("delayed:#{timestamp.to_i}").to_i
93
+ end
94
+
95
+ # Returns an array of delayed items for the given timestamp
96
+ def delayed_timestamp_peek(timestamp, start, count)
97
+ if 1 == count
98
+ r = list_range "delayed:#{timestamp.to_i}", start, count
99
+ r.nil? ? [] : [r]
100
+ else
101
+ list_range "delayed:#{timestamp.to_i}", start, count
102
+ end
103
+ end
104
+
105
+ # Returns the next delayed queue timestamp
106
+ # (don't call directly)
107
+ def next_delayed_timestamp(at_time = nil)
108
+ items = redis.zrangebyscore(
109
+ :delayed_queue_schedule, '-inf', (at_time || Time.now).to_i,
110
+ limit: [0, 1]
111
+ )
112
+ timestamp = items.nil? ? nil : Array(items).first
113
+ timestamp.to_i unless timestamp.nil?
114
+ end
115
+
116
+ # Returns the next item to be processed for a given timestamp, nil if
117
+ # done. (don't call directly)
118
+ # +timestamp+ can either be in seconds or a datetime
119
+ def next_item_for_timestamp(timestamp)
120
+ key = "delayed:#{timestamp.to_i}"
121
+
122
+ encoded_item = redis.lpop(key)
123
+ redis.srem("timestamps:#{encoded_item}", key)
124
+ item = decode(encoded_item)
125
+
126
+ # If the list is empty, remove it.
127
+ clean_up_timestamp(key, timestamp)
128
+ item
129
+ end
130
+
131
+ # Clears all jobs created with enqueue_at or enqueue_in
132
+ def reset_delayed_queue
133
+ Array(redis.zrange(:delayed_queue_schedule, 0, -1)).each do |item|
134
+ key = "delayed:#{item}"
135
+ items = redis.lrange(key, 0, -1)
136
+ redis.pipelined do
137
+ items.each { |ts_item| redis.del("timestamps:#{ts_item}") }
138
+ end
139
+ redis.del key
140
+ end
141
+
142
+ redis.del :delayed_queue_schedule
143
+ end
144
+
145
+ # Given an encoded item, remove it from the delayed_queue
146
+ def remove_delayed(klass, *args)
147
+ search = encode(job_to_hash(klass, args))
148
+ timestamps = redis.smembers("timestamps:#{search}")
149
+
150
+ replies = redis.pipelined do
151
+ timestamps.each do |key|
152
+ redis.lrem(key, 0, search)
153
+ redis.srem("timestamps:#{search}", key)
154
+ end
155
+ end
156
+
157
+ return 0 if replies.nil? || replies.empty?
158
+ replies.each_slice(2).map(&:first).inject(:+)
159
+ end
160
+
161
+ # Given an encoded item, enqueue it now
162
+ def enqueue_delayed(klass, *args)
163
+ hash = job_to_hash(klass, args)
164
+ remove_delayed(klass, *args).times do
165
+ Resque::Scheduler.enqueue_from_config(hash)
166
+ end
167
+ end
168
+
169
+ # Given a block, remove jobs that return true from a block
170
+ #
171
+ # This allows for removal of delayed jobs that have arguments matching
172
+ # certain criteria
173
+ def remove_delayed_selection
174
+ fail ArgumentError, 'Please supply a block' unless block_given?
175
+
176
+ destroyed = 0
177
+ # There is no way to search Redis list entries for a partial match,
178
+ # so we query for all delayed job tasks and do our matching after
179
+ # decoding the payload data
180
+ jobs = Resque.redis.keys('delayed:*')
181
+ jobs.each do |job|
182
+ index = Resque.redis.llen(job) - 1
183
+ while index >= 0
184
+ payload = Resque.redis.lindex(job, index)
185
+ decoded_payload = decode(payload)
186
+ if yield(decoded_payload['args'])
187
+ removed = redis.lrem job, 0, payload
188
+ destroyed += removed
189
+ index -= removed
190
+ else
191
+ index -= 1
192
+ end
193
+ end
194
+ end
195
+ destroyed
196
+ end
197
+
198
+ # Given a timestamp and job (klass + args) it removes all instances and
199
+ # returns the count of jobs removed.
200
+ #
201
+ # O(N) where N is the number of jobs scheduled to fire at the given
202
+ # timestamp
203
+ def remove_delayed_job_from_timestamp(timestamp, klass, *args)
204
+ key = "delayed:#{timestamp.to_i}"
205
+ encoded_job = encode(job_to_hash(klass, args))
206
+
207
+ redis.srem("timestamps:#{encoded_job}", key)
208
+ count = redis.lrem(key, 0, encoded_job)
209
+ clean_up_timestamp(key, timestamp)
210
+
211
+ count
212
+ end
213
+
214
+ def count_all_scheduled_jobs
215
+ total_jobs = 0
216
+ Array(redis.zrange(:delayed_queue_schedule, 0, -1)).each do |ts|
217
+ total_jobs += redis.llen("delayed:#{ts}").to_i
218
+ end
219
+ total_jobs
220
+ end
221
+
222
+ # Discover if a job has been delayed.
223
+ # Examples
224
+ # Resque.delayed?(MyJob)
225
+ # Resque.delayed?(MyJob, id: 1)
226
+ # Returns true if the job has been delayed
227
+ def delayed?(klass, *args)
228
+ !scheduled_at(klass, *args).empty?
229
+ end
230
+
231
+ # Returns delayed jobs schedule timestamp for +klass+, +args+.
232
+ def scheduled_at(klass, *args)
233
+ search = encode(job_to_hash(klass, args))
234
+ redis.smembers("timestamps:#{search}").map do |key|
235
+ key.tr('delayed:', '').to_i
236
+ end
237
+ end
238
+
239
+ def last_enqueued_at(job_name, date)
240
+ redis.hset('delayed:last_enqueued_at', job_name, date)
241
+ end
242
+
243
+ def get_last_enqueued_at(job_name)
244
+ redis.hget('delayed:last_enqueued_at', job_name)
245
+ end
246
+
247
+ private
248
+
249
+ def job_to_hash(klass, args)
250
+ { class: klass.to_s, args: args, queue: queue_from_class(klass) }
251
+ end
252
+
253
+ def job_to_hash_with_queue(queue, klass, args)
254
+ { class: klass.to_s, args: args, queue: queue }
255
+ end
256
+
257
+ def clean_up_timestamp(key, timestamp)
258
+ # If the list is empty, remove it.
259
+
260
+ # Use a watch here to ensure nobody adds jobs to this delayed
261
+ # queue while we're removing it.
262
+ redis.watch key
263
+ if 0 == redis.llen(key).to_i
264
+ redis.multi do
265
+ redis.del key
266
+ redis.zrem :delayed_queue_schedule, timestamp.to_i
267
+ end
268
+ else
269
+ redis.unwatch
270
+ end
271
+ end
272
+
273
+ def plugin
274
+ Resque::Scheduler::Plugin
275
+ end
276
+ end
277
+ end
278
+ end