resque-scheduler 2.3.1 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of resque-scheduler might be problematic. Click here for more details.
- data/.gitignore +3 -0
- data/.rubocop.yml +11 -11
- data/.simplecov +1 -0
- data/.travis.yml +5 -2
- data/AUTHORS.md +3 -0
- data/HISTORY.md +26 -2
- data/LICENSE +1 -1
- data/README.md +120 -31
- data/ROADMAP.md +10 -0
- data/Rakefile +7 -19
- data/bin/resque-scheduler +5 -0
- data/examples/Rakefile +2 -0
- data/examples/config/initializers/resque-web.rb +37 -0
- data/examples/dynamic-scheduling/README.md +28 -0
- data/examples/dynamic-scheduling/app/jobs/fix_schedules_job.rb +54 -0
- data/examples/dynamic-scheduling/app/jobs/send_email_job.rb +9 -0
- data/examples/dynamic-scheduling/app/models/user.rb +16 -0
- data/examples/dynamic-scheduling/config/resque.yml +4 -0
- data/examples/dynamic-scheduling/config/static_schedule.yml +7 -0
- data/examples/dynamic-scheduling/lib/tasks/resque.rake +48 -0
- data/examples/run-resque-web +3 -0
- data/lib/resque-scheduler.rb +2 -0
- data/lib/resque/scheduler.rb +130 -41
- data/lib/resque/scheduler/lock/resilient.rb +1 -1
- data/lib/resque/scheduler_locking.rb +3 -1
- data/lib/resque_scheduler.rb +73 -31
- data/lib/resque_scheduler/cli.rb +160 -0
- data/lib/resque_scheduler/logger_builder.rb +27 -8
- data/lib/resque_scheduler/plugin.rb +10 -7
- data/lib/resque_scheduler/server.rb +52 -11
- data/lib/resque_scheduler/server/views/delayed.erb +2 -0
- data/lib/resque_scheduler/server/views/delayed_schedules.erb +20 -0
- data/lib/resque_scheduler/server/views/scheduler.erb +4 -12
- data/lib/resque_scheduler/tasks.rb +15 -27
- data/lib/resque_scheduler/version.rb +1 -1
- data/resque-scheduler.gemspec +2 -0
- data/test/cli_test.rb +286 -0
- data/test/delayed_queue_test.rb +70 -1
- data/test/resque-web_test.rb +36 -1
- data/test/scheduler_args_test.rb +51 -17
- data/test/scheduler_hooks_test.rb +1 -1
- data/test/scheduler_locking_test.rb +63 -1
- data/test/scheduler_setup_test.rb +54 -18
- data/test/scheduler_task_test.rb +35 -0
- data/test/scheduler_test.rb +130 -42
- data/test/support/redis_instance.rb +8 -3
- data/test/test_helper.rb +47 -20
- metadata +77 -6
- checksums.yaml +0 -15
data/examples/Rakefile
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
# vim:fileencoding=utf-8
|
2
|
+
|
3
|
+
require 'json'
|
4
|
+
require 'yaml'
|
5
|
+
require 'resque'
|
6
|
+
|
7
|
+
redis_env_var = ENV['REDIS_PROVIDER'] || 'REDIS_URL'
|
8
|
+
Resque.redis = ENV[redis_env_var] || 'localhost:6379'
|
9
|
+
|
10
|
+
require 'resque_scheduler'
|
11
|
+
require 'resque_scheduler/server'
|
12
|
+
|
13
|
+
schedule_yml = ENV['RESQUE_SCHEDULE_YML']
|
14
|
+
if schedule_yml
|
15
|
+
if File.exist?(schedule_yml)
|
16
|
+
Resque.schedule = YAML.load_file(schedule_yml)
|
17
|
+
else
|
18
|
+
Resque.schedule = YAML.load(schedule_yml)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
schedule_json = ENV['RESQUE_SCHEDULE_JSON']
|
23
|
+
if schedule_json
|
24
|
+
if File.exist?(schedule_json)
|
25
|
+
Resque.schedule = JSON.parse(File.read(schedule_json))
|
26
|
+
else
|
27
|
+
Resque.schedule = JSON.parse(schedule_json)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
class Putter
|
32
|
+
@queue = 'putting'
|
33
|
+
|
34
|
+
def self.perform(*args)
|
35
|
+
args.each { |arg| puts arg }
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
Dynamic Scheduling Example
|
2
|
+
==========================
|
3
|
+
|
4
|
+
Possible workaround for
|
5
|
+
https://github.com/resque/resque-scheduler/issues/269
|
6
|
+
|
7
|
+
This folder contains just the relevant files you would have to put into
|
8
|
+
a rails application.
|
9
|
+
|
10
|
+
The problem we want to fix is that when resque-scheduler is restarted,
|
11
|
+
any dynamically added jobs are wiped. To fix it, we will run a
|
12
|
+
statically scheduled job that dynamically reschedules any missing
|
13
|
+
dynamic schedules.
|
14
|
+
|
15
|
+
This workaround uses both a dynamic schedule (every time a user is
|
16
|
+
created, a schedule is dynamically added to send him a daily email) and
|
17
|
+
a static schedule (a job runs hourly, starting 10 seconds after starting
|
18
|
+
resque-scheduler, to check that there is a scheduled job to send an
|
19
|
+
email for every user; missing schedules are added).
|
20
|
+
|
21
|
+
This way even though a resque-scheduler restart wipes all dynamic
|
22
|
+
schedules, they are recreated by the `fix_schedules` job that runs in
|
23
|
+
the static schedule. Even if dynamic schedules were lost for any reason
|
24
|
+
(data loss in redis clusters, whatever), they will be recreated hourly.
|
25
|
+
|
26
|
+
This workaround requires that enough information is saved in the
|
27
|
+
database to recreate all dynamic schedules. In this case we create one
|
28
|
+
dynamically scheduled job for every user in the database.
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# vim:fileencoding=utf-8
|
2
|
+
#
|
3
|
+
# Background job to fix the schedule for email sending. Any missing
|
4
|
+
# schedule will be added to resque-schedule.
|
5
|
+
#
|
6
|
+
# Recent resque-scheduler versions wipe all dynamic schedules when
|
7
|
+
# restarting. This means all dynamic schedules, which are added via the
|
8
|
+
# API, are wiped on each application redeployment. A workaround for this
|
9
|
+
# sometimes undesirable behavior is to make this job part of a static
|
10
|
+
# schedule (see config/initializers/resque.rb and
|
11
|
+
# config/static_schedule.yml). This job will be scheduled to run every
|
12
|
+
# hour even after restarting resque-schedule, and will add back the
|
13
|
+
# dynamic schedules that were wiped on restart. It also serves as
|
14
|
+
# safeguard against schedules getting lost for any reason.
|
15
|
+
#
|
16
|
+
# For more detail about this unfortunate behavior of resque-scheduler see:
|
17
|
+
#
|
18
|
+
# https://github.com/resque/resque-scheduler/issues/269
|
19
|
+
#
|
20
|
+
# The perform method of this class will be invoked from a Resque worker.
|
21
|
+
|
22
|
+
class FixSchedulesJob
|
23
|
+
@queue = :send_emails
|
24
|
+
|
25
|
+
# Fix email sending schedules. Any user which does not have scheduled
|
26
|
+
# sending of emails will be detected, and the missing scheduled job
|
27
|
+
# will be added to resque-schedule.
|
28
|
+
#
|
29
|
+
# This method is intended to be invoked from Resque, which means it is
|
30
|
+
# performed in the background.
|
31
|
+
def self.perform
|
32
|
+
users_unscheduled = []
|
33
|
+
|
34
|
+
User.all.each do |user|
|
35
|
+
# get schedule for the user
|
36
|
+
schedule = Resque.get_schedule("send_email_#{user.id}")
|
37
|
+
# if a user has no schedule, add it to the array
|
38
|
+
if schedule.nil?
|
39
|
+
users_unscheduled << user
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
if users_unscheduled.length > 0
|
44
|
+
users_unscheduled.each do |user|
|
45
|
+
name = "send_email_#{user.id}"
|
46
|
+
config = {}
|
47
|
+
config[:class] = 'SendEmailJob'
|
48
|
+
config[:args] = user.id
|
49
|
+
config[:every] = '1d'
|
50
|
+
Resque.set_schedule(name, config)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# vim:fileencoding=utf-8
|
2
|
+
|
3
|
+
class User < ActiveRecord::Base
|
4
|
+
after_create :schedule_send_email
|
5
|
+
|
6
|
+
private
|
7
|
+
|
8
|
+
def schedule_send_email
|
9
|
+
name = "send_email_#{id}"
|
10
|
+
config = {}
|
11
|
+
config[:class] = 'SendEmailJob'
|
12
|
+
config[:args] = id
|
13
|
+
config[:every] = '1d'
|
14
|
+
Resque.set_schedule(name, config)
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
# vim:fileencoding=utf-8
|
2
|
+
|
3
|
+
require 'resque/tasks'
|
4
|
+
require 'resque_scheduler/tasks'
|
5
|
+
require 'yaml'
|
6
|
+
|
7
|
+
namespace :resque do
|
8
|
+
task :setup do
|
9
|
+
require 'resque'
|
10
|
+
require 'resque_scheduler'
|
11
|
+
|
12
|
+
rails_root = ENV['RAILS_ROOT'] || File.expand_path('../../../', __FILE__)
|
13
|
+
rails_env = ENV['RAILS_ENV'] || 'development'
|
14
|
+
|
15
|
+
# In resque-only servers we must require each job class individually,
|
16
|
+
# because we're not running the full Rails app
|
17
|
+
require "#{rails_root}/app/jobs/send_email_job"
|
18
|
+
require "#{rails_root}/app/jobs/fix_schedules_job"
|
19
|
+
|
20
|
+
resque_config = YAML.load_file(
|
21
|
+
File.join(rails_root.to_s, 'config', 'resque.yml')
|
22
|
+
)
|
23
|
+
Resque.redis = resque_config[rails_env]
|
24
|
+
|
25
|
+
# If you want to be able to dynamically change the schedule,
|
26
|
+
# uncomment this line. A dynamic schedule can be updated via the
|
27
|
+
# Resque::Scheduler.set_schedule (and remove_schedule) methods.
|
28
|
+
# When dynamic is set to true, the scheduler process looks for
|
29
|
+
# schedule changes and applies them on the fly.
|
30
|
+
# Note: This feature is only available in >=2.0.0.
|
31
|
+
Resque::Scheduler.dynamic = true
|
32
|
+
|
33
|
+
# Load static schedule (only in background servers).
|
34
|
+
# The schedule doesn't need to be stored in a YAML, it just needs to
|
35
|
+
# be a hash. YAML is usually the easiest.
|
36
|
+
Resque.schedule = YAML.load_file(
|
37
|
+
File.join(rails_root.to_s, 'config', 'static_schedule.yml')
|
38
|
+
)
|
39
|
+
|
40
|
+
Resque.before_fork do |job|
|
41
|
+
# Reconnect to the DB before running each job. Otherwise we get errors if
|
42
|
+
# the DB is restarted after starting Resque.
|
43
|
+
# Absolutely necessary on Heroku, otherwise we get a "PG::Error: SSL
|
44
|
+
# SYSCALL error: EOF detected" exception
|
45
|
+
ActiveRecord::Base.establish_connection
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
data/lib/resque/scheduler.rb
CHANGED
@@ -3,44 +3,92 @@ require 'resque/scheduler_locking'
|
|
3
3
|
require 'resque_scheduler/logger_builder'
|
4
4
|
|
5
5
|
module Resque
|
6
|
-
|
7
6
|
class Scheduler
|
8
|
-
|
9
7
|
extend Resque::SchedulerLocking
|
10
8
|
|
11
9
|
class << self
|
10
|
+
# Allows for block-style configuration
|
11
|
+
def configure
|
12
|
+
yield self
|
13
|
+
end
|
14
|
+
|
15
|
+
# Used in `#load_schedule_job`
|
16
|
+
attr_writer :env
|
17
|
+
|
18
|
+
def env
|
19
|
+
return @env if @env
|
20
|
+
@env ||= Rails.env if defined?(Rails)
|
21
|
+
@env ||= ENV['RAILS_ENV']
|
22
|
+
@env
|
23
|
+
end
|
12
24
|
|
13
25
|
# If true, logs more stuff...
|
14
|
-
|
26
|
+
attr_writer :verbose
|
27
|
+
|
28
|
+
def verbose
|
29
|
+
@verbose ||= !!ENV['VERBOSE']
|
30
|
+
end
|
15
31
|
|
16
32
|
# If set, produces no output
|
17
|
-
|
33
|
+
attr_writer :mute
|
34
|
+
|
35
|
+
def mute
|
36
|
+
@mute ||= !!ENV['MUTE']
|
37
|
+
end
|
18
38
|
|
19
39
|
# If set, will write messages to the file
|
20
|
-
|
40
|
+
attr_writer :logfile
|
41
|
+
|
42
|
+
def logfile
|
43
|
+
@logfile ||= ENV['LOGFILE']
|
44
|
+
end
|
45
|
+
|
46
|
+
# Sets whether to log in 'text' or 'json'
|
47
|
+
attr_writer :logformat
|
48
|
+
|
49
|
+
def logformat
|
50
|
+
@logformat ||= ENV['LOGFORMAT']
|
51
|
+
end
|
21
52
|
|
22
53
|
# If set, will try to update the schedule in the loop
|
23
|
-
|
54
|
+
attr_writer :dynamic
|
55
|
+
|
56
|
+
def dynamic
|
57
|
+
@dynamic ||= !!ENV['DYNAMIC_SCHEDULE']
|
58
|
+
end
|
59
|
+
|
60
|
+
# If set, will append the app name to procline
|
61
|
+
attr_writer :app_name
|
62
|
+
|
63
|
+
def app_name
|
64
|
+
@app_name ||= ENV['APP_NAME']
|
65
|
+
end
|
24
66
|
|
25
67
|
# Amount of time in seconds to sleep between polls of the delayed
|
26
68
|
# queue. Defaults to 5
|
27
69
|
attr_writer :poll_sleep_amount
|
28
70
|
|
71
|
+
def poll_sleep_amount
|
72
|
+
@poll_sleep_amount ||=
|
73
|
+
Float(ENV.fetch('RESQUE_SCHEDULER_INTERVAL', '5'))
|
74
|
+
end
|
75
|
+
|
29
76
|
attr_writer :logger
|
30
77
|
|
78
|
+
def logger
|
79
|
+
@logger ||= ResqueScheduler::LoggerBuilder.new(
|
80
|
+
:mute => mute,
|
81
|
+
:verbose => verbose,
|
82
|
+
:log_dev => logfile,
|
83
|
+
:format => logformat
|
84
|
+
).build
|
85
|
+
end
|
86
|
+
|
31
87
|
# the Rufus::Scheduler jobs that are scheduled
|
32
88
|
def scheduled_jobs
|
33
89
|
@@scheduled_jobs
|
34
90
|
end
|
35
91
|
|
36
|
-
def poll_sleep_amount
|
37
|
-
@poll_sleep_amount ||= 5 # seconds
|
38
|
-
end
|
39
|
-
|
40
|
-
def logger
|
41
|
-
@logger ||= ResqueScheduler::LoggerBuilder.new(:mute => mute, :verbose => verbose, :log_dev => logfile).build
|
42
|
-
end
|
43
|
-
|
44
92
|
# Schedule all jobs and continually look for delayed jobs (never returns)
|
45
93
|
def run
|
46
94
|
$0 = "resque-scheduler: Starting"
|
@@ -62,6 +110,8 @@ module Resque
|
|
62
110
|
load_schedule!
|
63
111
|
end
|
64
112
|
|
113
|
+
@th = Thread.current
|
114
|
+
|
65
115
|
# Now start the scheduling part of the loop.
|
66
116
|
loop do
|
67
117
|
if is_master?
|
@@ -78,7 +128,6 @@ module Resque
|
|
78
128
|
# never gets here.
|
79
129
|
end
|
80
130
|
|
81
|
-
|
82
131
|
# For all signals, set the shutdown flag and wait for current
|
83
132
|
# poll/enqueing to finish (should be almost istant). In the
|
84
133
|
# case of sleeping, exit immediately.
|
@@ -139,13 +188,17 @@ module Resque
|
|
139
188
|
args
|
140
189
|
end
|
141
190
|
|
142
|
-
# Loads a job schedule into the Rufus::Scheduler and stores it in
|
191
|
+
# Loads a job schedule into the Rufus::Scheduler and stores it in
|
192
|
+
# @@scheduled_jobs
|
143
193
|
def load_schedule_job(name, config)
|
144
|
-
# If rails_env is set in the config,
|
145
|
-
#
|
146
|
-
# job should be scheduled regardless of
|
147
|
-
#
|
148
|
-
|
194
|
+
# If `rails_env` or `env` is set in the config, load jobs only if they
|
195
|
+
# are meant to be loaded in `Resque::Scheduler.env`. If `rails_env` or
|
196
|
+
# `env` is missing, the job should be scheduled regardless of the value
|
197
|
+
# of `Resque::Scheduler.env`.
|
198
|
+
|
199
|
+
configured_env = config['rails_env'] || config['env']
|
200
|
+
|
201
|
+
if configured_env.nil? || env_matches?(configured_env)
|
149
202
|
log! "Scheduling #{name} "
|
150
203
|
interval_defined = false
|
151
204
|
interval_types = %w{cron every}
|
@@ -165,13 +218,25 @@ module Resque
|
|
165
218
|
unless interval_defined
|
166
219
|
log! "no #{interval_types.join(' / ')} found for #{config['class']} (#{name}) - skipping"
|
167
220
|
end
|
221
|
+
else
|
222
|
+
log "Skipping schedule of #{name} because configured " <<
|
223
|
+
"env #{configured_env.inspect} does not match current " <<
|
224
|
+
"env #{env.inspect}"
|
168
225
|
end
|
169
226
|
end
|
170
227
|
|
171
|
-
# Returns true if the given schedule config hash matches the current
|
172
|
-
# ENV['RAILS_ENV']
|
228
|
+
# Returns true if the given schedule config hash matches the current env
|
173
229
|
def rails_env_matches?(config)
|
174
|
-
|
230
|
+
warn '`Resque::Scheduler.rails_env_matches?` is deprecated. ' <<
|
231
|
+
'Please use `Resque::Scheduler.env_matches?` instead.'
|
232
|
+
config['rails_env'] && env &&
|
233
|
+
config['rails_env'].split(/[\s,]+/).include?(env)
|
234
|
+
end
|
235
|
+
|
236
|
+
# Returns true if the current env is non-nil and the configured env
|
237
|
+
# (which is a comma-split string) includes the current env.
|
238
|
+
def env_matches?(configured_env)
|
239
|
+
env && configured_env.split(/[\s,]+/).include?(env)
|
175
240
|
end
|
176
241
|
|
177
242
|
# Handles queueing delayed items
|
@@ -302,41 +367,65 @@ module Resque
|
|
302
367
|
|
303
368
|
# Sleeps and returns true
|
304
369
|
def poll_sleep
|
305
|
-
|
306
|
-
|
307
|
-
|
370
|
+
handle_shutdown do
|
371
|
+
begin
|
372
|
+
begin
|
373
|
+
@sleeping = true
|
374
|
+
sleep poll_sleep_amount
|
375
|
+
@sleeping = false
|
376
|
+
rescue Interrupt
|
377
|
+
if @shutdown
|
378
|
+
Resque.clean_schedules
|
379
|
+
release_master_lock!
|
380
|
+
end
|
381
|
+
end
|
382
|
+
ensure
|
383
|
+
@sleeping = false
|
384
|
+
end
|
385
|
+
end
|
308
386
|
true
|
309
387
|
end
|
310
388
|
|
311
389
|
# Sets the shutdown flag, clean schedules and exits if sleeping
|
312
390
|
def shutdown
|
391
|
+
return if @shutdown
|
313
392
|
@shutdown = true
|
314
|
-
|
315
|
-
if @sleeping
|
316
|
-
thread = Thread.new do
|
317
|
-
Resque.clean_schedules
|
318
|
-
release_master_lock!
|
319
|
-
end
|
320
|
-
thread.join
|
321
|
-
exit
|
322
|
-
end
|
393
|
+
log!('Shutting down')
|
394
|
+
@th.raise Interrupt if @sleeping
|
323
395
|
end
|
324
396
|
|
325
397
|
def log!(msg)
|
326
|
-
logger.info msg
|
398
|
+
logger.info { msg }
|
327
399
|
end
|
328
400
|
|
329
401
|
def log(msg)
|
330
|
-
logger.debug msg
|
402
|
+
logger.debug { msg }
|
331
403
|
end
|
332
404
|
|
333
405
|
def procline(string)
|
334
406
|
log! string
|
335
|
-
|
407
|
+
argv0 = build_procline(string)
|
408
|
+
log "Setting procline #{argv0.inspect}"
|
409
|
+
$0 = argv0
|
336
410
|
end
|
337
411
|
|
338
|
-
|
412
|
+
private
|
339
413
|
|
340
|
-
|
414
|
+
def app_str
|
415
|
+
app_name ? "[#{app_name}]" : ''
|
416
|
+
end
|
341
417
|
|
418
|
+
def env_str
|
419
|
+
env ? "[#{env}]" : ''
|
420
|
+
end
|
421
|
+
|
422
|
+
def build_procline(string)
|
423
|
+
"#{internal_name}#{app_str}#{env_str}: #{string}"
|
424
|
+
end
|
425
|
+
|
426
|
+
def internal_name
|
427
|
+
"resque-scheduler-#{ResqueScheduler::VERSION}"
|
428
|
+
end
|
429
|
+
end
|
430
|
+
end
|
342
431
|
end
|