resque-scheduler 2.3.1 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of resque-scheduler might be problematic. Click here for more details.

Files changed (49) hide show
  1. data/.gitignore +3 -0
  2. data/.rubocop.yml +11 -11
  3. data/.simplecov +1 -0
  4. data/.travis.yml +5 -2
  5. data/AUTHORS.md +3 -0
  6. data/HISTORY.md +26 -2
  7. data/LICENSE +1 -1
  8. data/README.md +120 -31
  9. data/ROADMAP.md +10 -0
  10. data/Rakefile +7 -19
  11. data/bin/resque-scheduler +5 -0
  12. data/examples/Rakefile +2 -0
  13. data/examples/config/initializers/resque-web.rb +37 -0
  14. data/examples/dynamic-scheduling/README.md +28 -0
  15. data/examples/dynamic-scheduling/app/jobs/fix_schedules_job.rb +54 -0
  16. data/examples/dynamic-scheduling/app/jobs/send_email_job.rb +9 -0
  17. data/examples/dynamic-scheduling/app/models/user.rb +16 -0
  18. data/examples/dynamic-scheduling/config/resque.yml +4 -0
  19. data/examples/dynamic-scheduling/config/static_schedule.yml +7 -0
  20. data/examples/dynamic-scheduling/lib/tasks/resque.rake +48 -0
  21. data/examples/run-resque-web +3 -0
  22. data/lib/resque-scheduler.rb +2 -0
  23. data/lib/resque/scheduler.rb +130 -41
  24. data/lib/resque/scheduler/lock/resilient.rb +1 -1
  25. data/lib/resque/scheduler_locking.rb +3 -1
  26. data/lib/resque_scheduler.rb +73 -31
  27. data/lib/resque_scheduler/cli.rb +160 -0
  28. data/lib/resque_scheduler/logger_builder.rb +27 -8
  29. data/lib/resque_scheduler/plugin.rb +10 -7
  30. data/lib/resque_scheduler/server.rb +52 -11
  31. data/lib/resque_scheduler/server/views/delayed.erb +2 -0
  32. data/lib/resque_scheduler/server/views/delayed_schedules.erb +20 -0
  33. data/lib/resque_scheduler/server/views/scheduler.erb +4 -12
  34. data/lib/resque_scheduler/tasks.rb +15 -27
  35. data/lib/resque_scheduler/version.rb +1 -1
  36. data/resque-scheduler.gemspec +2 -0
  37. data/test/cli_test.rb +286 -0
  38. data/test/delayed_queue_test.rb +70 -1
  39. data/test/resque-web_test.rb +36 -1
  40. data/test/scheduler_args_test.rb +51 -17
  41. data/test/scheduler_hooks_test.rb +1 -1
  42. data/test/scheduler_locking_test.rb +63 -1
  43. data/test/scheduler_setup_test.rb +54 -18
  44. data/test/scheduler_task_test.rb +35 -0
  45. data/test/scheduler_test.rb +130 -42
  46. data/test/support/redis_instance.rb +8 -3
  47. data/test/test_helper.rb +47 -20
  48. metadata +77 -6
  49. checksums.yaml +0 -15
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+ # vim:fileencoding=utf-8
3
+
4
+ require 'resque_scheduler'
5
+ ResqueScheduler::Cli.run!
@@ -0,0 +1,2 @@
1
+ # vim:fileencoding=utf-8
2
+ require 'resque_scheduler/tasks'
@@ -0,0 +1,37 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ require 'json'
4
+ require 'yaml'
5
+ require 'resque'
6
+
7
+ redis_env_var = ENV['REDIS_PROVIDER'] || 'REDIS_URL'
8
+ Resque.redis = ENV[redis_env_var] || 'localhost:6379'
9
+
10
+ require 'resque_scheduler'
11
+ require 'resque_scheduler/server'
12
+
13
+ schedule_yml = ENV['RESQUE_SCHEDULE_YML']
14
+ if schedule_yml
15
+ if File.exist?(schedule_yml)
16
+ Resque.schedule = YAML.load_file(schedule_yml)
17
+ else
18
+ Resque.schedule = YAML.load(schedule_yml)
19
+ end
20
+ end
21
+
22
+ schedule_json = ENV['RESQUE_SCHEDULE_JSON']
23
+ if schedule_json
24
+ if File.exist?(schedule_json)
25
+ Resque.schedule = JSON.parse(File.read(schedule_json))
26
+ else
27
+ Resque.schedule = JSON.parse(schedule_json)
28
+ end
29
+ end
30
+
31
+ class Putter
32
+ @queue = 'putting'
33
+
34
+ def self.perform(*args)
35
+ args.each { |arg| puts arg }
36
+ end
37
+ end
@@ -0,0 +1,28 @@
1
+ Dynamic Scheduling Example
2
+ ==========================
3
+
4
+ Possible workaround for
5
+ https://github.com/resque/resque-scheduler/issues/269
6
+
7
+ This folder contains just the relevant files you would have to put into
8
+ a rails application.
9
+
10
+ The problem we want to fix is that when resque-scheduler is restarted,
11
+ any dynamically added jobs are wiped. To fix it, we will run a
12
+ statically scheduled job that dynamically reschedules any missing
13
+ dynamic schedules.
14
+
15
+ This workaround uses both a dynamic schedule (every time a user is
16
+ created, a schedule is dynamically added to send him a daily email) and
17
+ a static schedule (a job runs hourly, starting 10 seconds after starting
18
+ resque-scheduler, to check that there is a scheduled job to send an
19
+ email for every user; missing schedules are added).
20
+
21
+ This way even though a resque-scheduler restart wipes all dynamic
22
+ schedules, they are recreated by the `fix_schedules` job that runs in
23
+ the static schedule. Even if dynamic schedules were lost for any reason
24
+ (data loss in redis clusters, whatever), they will be recreated hourly.
25
+
26
+ This workaround requires that enough information is saved in the
27
+ database to recreate all dynamic schedules. In this case we create one
28
+ dynamically scheduled job for every user in the database.
@@ -0,0 +1,54 @@
1
+ # vim:fileencoding=utf-8
2
+ #
3
+ # Background job to fix the schedule for email sending. Any missing
4
+ # schedule will be added to resque-schedule.
5
+ #
6
+ # Recent resque-scheduler versions wipe all dynamic schedules when
7
+ # restarting. This means all dynamic schedules, which are added via the
8
+ # API, are wiped on each application redeployment. A workaround for this
9
+ # sometimes undesirable behavior is to make this job part of a static
10
+ # schedule (see config/initializers/resque.rb and
11
+ # config/static_schedule.yml). This job will be scheduled to run every
12
+ # hour even after restarting resque-schedule, and will add back the
13
+ # dynamic schedules that were wiped on restart. It also serves as
14
+ # safeguard against schedules getting lost for any reason.
15
+ #
16
+ # For more detail about this unfortunate behavior of resque-scheduler see:
17
+ #
18
+ # https://github.com/resque/resque-scheduler/issues/269
19
+ #
20
+ # The perform method of this class will be invoked from a Resque worker.
21
+
22
+ class FixSchedulesJob
23
+ @queue = :send_emails
24
+
25
+ # Fix email sending schedules. Any user which does not have scheduled
26
+ # sending of emails will be detected, and the missing scheduled job
27
+ # will be added to resque-schedule.
28
+ #
29
+ # This method is intended to be invoked from Resque, which means it is
30
+ # performed in the background.
31
+ def self.perform
32
+ users_unscheduled = []
33
+
34
+ User.all.each do |user|
35
+ # get schedule for the user
36
+ schedule = Resque.get_schedule("send_email_#{user.id}")
37
+ # if a user has no schedule, add it to the array
38
+ if schedule.nil?
39
+ users_unscheduled << user
40
+ end
41
+ end
42
+
43
+ if users_unscheduled.length > 0
44
+ users_unscheduled.each do |user|
45
+ name = "send_email_#{user.id}"
46
+ config = {}
47
+ config[:class] = 'SendEmailJob'
48
+ config[:args] = user.id
49
+ config[:every] = '1d'
50
+ Resque.set_schedule(name, config)
51
+ end
52
+ end
53
+ end
54
+ end
@@ -0,0 +1,9 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ class SendEmailJob
4
+ @queue = :send_emails
5
+
6
+ def self.perform(user_id)
7
+ # ... do whatever you have to do to send an email to the user
8
+ end
9
+ end
@@ -0,0 +1,16 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ class User < ActiveRecord::Base
4
+ after_create :schedule_send_email
5
+
6
+ private
7
+
8
+ def schedule_send_email
9
+ name = "send_email_#{id}"
10
+ config = {}
11
+ config[:class] = 'SendEmailJob'
12
+ config[:args] = id
13
+ config[:every] = '1d'
14
+ Resque.set_schedule(name, config)
15
+ end
16
+ end
@@ -0,0 +1,4 @@
1
+ ---
2
+ development: localhost:6379
3
+ test: localhost:6379:1
4
+ production: localhost:6379 # or wherever your redis-server is in production
@@ -0,0 +1,7 @@
1
+ ---
2
+ FixSchedulesJob:
3
+ description: 'Add any missing email sending schedules'
4
+ queue: send_emails
5
+ every:
6
+ - '1h'
7
+ - :first_in: '10s'
@@ -0,0 +1,48 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ require 'resque/tasks'
4
+ require 'resque_scheduler/tasks'
5
+ require 'yaml'
6
+
7
+ namespace :resque do
8
+ task :setup do
9
+ require 'resque'
10
+ require 'resque_scheduler'
11
+
12
+ rails_root = ENV['RAILS_ROOT'] || File.expand_path('../../../', __FILE__)
13
+ rails_env = ENV['RAILS_ENV'] || 'development'
14
+
15
+ # In resque-only servers we must require each job class individually,
16
+ # because we're not running the full Rails app
17
+ require "#{rails_root}/app/jobs/send_email_job"
18
+ require "#{rails_root}/app/jobs/fix_schedules_job"
19
+
20
+ resque_config = YAML.load_file(
21
+ File.join(rails_root.to_s, 'config', 'resque.yml')
22
+ )
23
+ Resque.redis = resque_config[rails_env]
24
+
25
+ # If you want to be able to dynamically change the schedule,
26
+ # uncomment this line. A dynamic schedule can be updated via the
27
+ # Resque::Scheduler.set_schedule (and remove_schedule) methods.
28
+ # When dynamic is set to true, the scheduler process looks for
29
+ # schedule changes and applies them on the fly.
30
+ # Note: This feature is only available in >=2.0.0.
31
+ Resque::Scheduler.dynamic = true
32
+
33
+ # Load static schedule (only in background servers).
34
+ # The schedule doesn't need to be stored in a YAML, it just needs to
35
+ # be a hash. YAML is usually the easiest.
36
+ Resque.schedule = YAML.load_file(
37
+ File.join(rails_root.to_s, 'config', 'static_schedule.yml')
38
+ )
39
+
40
+ Resque.before_fork do |job|
41
+ # Reconnect to the DB before running each job. Otherwise we get errors if
42
+ # the DB is restarted after starting Resque.
43
+ # Absolutely necessary on Heroku, otherwise we get a "PG::Error: SSL
44
+ # SYSCALL error: EOF detected" exception
45
+ ActiveRecord::Base.establish_connection
46
+ end
47
+ end
48
+ end
@@ -0,0 +1,3 @@
1
+ #!/bin/bash
2
+ cd "$(dirname $0)"
3
+ exec resque-web --foreground --no-launch config/initializers/resque-web.rb
@@ -0,0 +1,2 @@
1
+ # vim:fileencoding=utf-8
2
+ require File.expand_path('../resque_scheduler', __FILE__)
@@ -3,44 +3,92 @@ require 'resque/scheduler_locking'
3
3
  require 'resque_scheduler/logger_builder'
4
4
 
5
5
  module Resque
6
-
7
6
  class Scheduler
8
-
9
7
  extend Resque::SchedulerLocking
10
8
 
11
9
  class << self
10
+ # Allows for block-style configuration
11
+ def configure
12
+ yield self
13
+ end
14
+
15
+ # Used in `#load_schedule_job`
16
+ attr_writer :env
17
+
18
+ def env
19
+ return @env if @env
20
+ @env ||= Rails.env if defined?(Rails)
21
+ @env ||= ENV['RAILS_ENV']
22
+ @env
23
+ end
12
24
 
13
25
  # If true, logs more stuff...
14
- attr_accessor :verbose
26
+ attr_writer :verbose
27
+
28
+ def verbose
29
+ @verbose ||= !!ENV['VERBOSE']
30
+ end
15
31
 
16
32
  # If set, produces no output
17
- attr_accessor :mute
33
+ attr_writer :mute
34
+
35
+ def mute
36
+ @mute ||= !!ENV['MUTE']
37
+ end
18
38
 
19
39
  # If set, will write messages to the file
20
- attr_accessor :logfile
40
+ attr_writer :logfile
41
+
42
+ def logfile
43
+ @logfile ||= ENV['LOGFILE']
44
+ end
45
+
46
+ # Sets whether to log in 'text' or 'json'
47
+ attr_writer :logformat
48
+
49
+ def logformat
50
+ @logformat ||= ENV['LOGFORMAT']
51
+ end
21
52
 
22
53
  # If set, will try to update the schedule in the loop
23
- attr_accessor :dynamic
54
+ attr_writer :dynamic
55
+
56
+ def dynamic
57
+ @dynamic ||= !!ENV['DYNAMIC_SCHEDULE']
58
+ end
59
+
60
+ # If set, will append the app name to procline
61
+ attr_writer :app_name
62
+
63
+ def app_name
64
+ @app_name ||= ENV['APP_NAME']
65
+ end
24
66
 
25
67
  # Amount of time in seconds to sleep between polls of the delayed
26
68
  # queue. Defaults to 5
27
69
  attr_writer :poll_sleep_amount
28
70
 
71
+ def poll_sleep_amount
72
+ @poll_sleep_amount ||=
73
+ Float(ENV.fetch('RESQUE_SCHEDULER_INTERVAL', '5'))
74
+ end
75
+
29
76
  attr_writer :logger
30
77
 
78
+ def logger
79
+ @logger ||= ResqueScheduler::LoggerBuilder.new(
80
+ :mute => mute,
81
+ :verbose => verbose,
82
+ :log_dev => logfile,
83
+ :format => logformat
84
+ ).build
85
+ end
86
+
31
87
  # the Rufus::Scheduler jobs that are scheduled
32
88
  def scheduled_jobs
33
89
  @@scheduled_jobs
34
90
  end
35
91
 
36
- def poll_sleep_amount
37
- @poll_sleep_amount ||= 5 # seconds
38
- end
39
-
40
- def logger
41
- @logger ||= ResqueScheduler::LoggerBuilder.new(:mute => mute, :verbose => verbose, :log_dev => logfile).build
42
- end
43
-
44
92
  # Schedule all jobs and continually look for delayed jobs (never returns)
45
93
  def run
46
94
  $0 = "resque-scheduler: Starting"
@@ -62,6 +110,8 @@ module Resque
62
110
  load_schedule!
63
111
  end
64
112
 
113
+ @th = Thread.current
114
+
65
115
  # Now start the scheduling part of the loop.
66
116
  loop do
67
117
  if is_master?
@@ -78,7 +128,6 @@ module Resque
78
128
  # never gets here.
79
129
  end
80
130
 
81
-
82
131
  # For all signals, set the shutdown flag and wait for current
83
132
  # poll/enqueing to finish (should be almost istant). In the
84
133
  # case of sleeping, exit immediately.
@@ -139,13 +188,17 @@ module Resque
139
188
  args
140
189
  end
141
190
 
142
- # Loads a job schedule into the Rufus::Scheduler and stores it in @@scheduled_jobs
191
+ # Loads a job schedule into the Rufus::Scheduler and stores it in
192
+ # @@scheduled_jobs
143
193
  def load_schedule_job(name, config)
144
- # If rails_env is set in the config, enforce ENV['RAILS_ENV'] as
145
- # required for the jobs to be scheduled. If rails_env is missing, the
146
- # job should be scheduled regardless of what ENV['RAILS_ENV'] is set
147
- # to.
148
- if config['rails_env'].nil? || rails_env_matches?(config)
194
+ # If `rails_env` or `env` is set in the config, load jobs only if they
195
+ # are meant to be loaded in `Resque::Scheduler.env`. If `rails_env` or
196
+ # `env` is missing, the job should be scheduled regardless of the value
197
+ # of `Resque::Scheduler.env`.
198
+
199
+ configured_env = config['rails_env'] || config['env']
200
+
201
+ if configured_env.nil? || env_matches?(configured_env)
149
202
  log! "Scheduling #{name} "
150
203
  interval_defined = false
151
204
  interval_types = %w{cron every}
@@ -165,13 +218,25 @@ module Resque
165
218
  unless interval_defined
166
219
  log! "no #{interval_types.join(' / ')} found for #{config['class']} (#{name}) - skipping"
167
220
  end
221
+ else
222
+ log "Skipping schedule of #{name} because configured " <<
223
+ "env #{configured_env.inspect} does not match current " <<
224
+ "env #{env.inspect}"
168
225
  end
169
226
  end
170
227
 
171
- # Returns true if the given schedule config hash matches the current
172
- # ENV['RAILS_ENV']
228
+ # Returns true if the given schedule config hash matches the current env
173
229
  def rails_env_matches?(config)
174
- config['rails_env'] && ENV['RAILS_ENV'] && config['rails_env'].gsub(/\s/,'').split(',').include?(ENV['RAILS_ENV'])
230
+ warn '`Resque::Scheduler.rails_env_matches?` is deprecated. ' <<
231
+ 'Please use `Resque::Scheduler.env_matches?` instead.'
232
+ config['rails_env'] && env &&
233
+ config['rails_env'].split(/[\s,]+/).include?(env)
234
+ end
235
+
236
+ # Returns true if the current env is non-nil and the configured env
237
+ # (which is a comma-split string) includes the current env.
238
+ def env_matches?(configured_env)
239
+ env && configured_env.split(/[\s,]+/).include?(env)
175
240
  end
176
241
 
177
242
  # Handles queueing delayed items
@@ -302,41 +367,65 @@ module Resque
302
367
 
303
368
  # Sleeps and returns true
304
369
  def poll_sleep
305
- @sleeping = true
306
- handle_shutdown { sleep poll_sleep_amount }
307
- @sleeping = false
370
+ handle_shutdown do
371
+ begin
372
+ begin
373
+ @sleeping = true
374
+ sleep poll_sleep_amount
375
+ @sleeping = false
376
+ rescue Interrupt
377
+ if @shutdown
378
+ Resque.clean_schedules
379
+ release_master_lock!
380
+ end
381
+ end
382
+ ensure
383
+ @sleeping = false
384
+ end
385
+ end
308
386
  true
309
387
  end
310
388
 
311
389
  # Sets the shutdown flag, clean schedules and exits if sleeping
312
390
  def shutdown
391
+ return if @shutdown
313
392
  @shutdown = true
314
-
315
- if @sleeping
316
- thread = Thread.new do
317
- Resque.clean_schedules
318
- release_master_lock!
319
- end
320
- thread.join
321
- exit
322
- end
393
+ log!('Shutting down')
394
+ @th.raise Interrupt if @sleeping
323
395
  end
324
396
 
325
397
  def log!(msg)
326
- logger.info msg
398
+ logger.info { msg }
327
399
  end
328
400
 
329
401
  def log(msg)
330
- logger.debug msg
402
+ logger.debug { msg }
331
403
  end
332
404
 
333
405
  def procline(string)
334
406
  log! string
335
- $0 = "resque-scheduler-#{ResqueScheduler::VERSION}: #{string}"
407
+ argv0 = build_procline(string)
408
+ log "Setting procline #{argv0.inspect}"
409
+ $0 = argv0
336
410
  end
337
411
 
338
- end
412
+ private
339
413
 
340
- end
414
+ def app_str
415
+ app_name ? "[#{app_name}]" : ''
416
+ end
341
417
 
418
+ def env_str
419
+ env ? "[#{env}]" : ''
420
+ end
421
+
422
+ def build_procline(string)
423
+ "#{internal_name}#{app_str}#{env_str}: #{string}"
424
+ end
425
+
426
+ def internal_name
427
+ "resque-scheduler-#{ResqueScheduler::VERSION}"
428
+ end
429
+ end
430
+ end
342
431
  end