resque-admin-scheduler 1.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (40) hide show
  1. checksums.yaml +7 -0
  2. data/AUTHORS.md +81 -0
  3. data/CHANGELOG.md +456 -0
  4. data/CODE_OF_CONDUCT.md +74 -0
  5. data/CONTRIBUTING.md +6 -0
  6. data/Gemfile +4 -0
  7. data/LICENSE +23 -0
  8. data/README.md +691 -0
  9. data/Rakefile +21 -0
  10. data/exe/resque-scheduler +5 -0
  11. data/lib/resque/scheduler/cli.rb +147 -0
  12. data/lib/resque/scheduler/configuration.rb +73 -0
  13. data/lib/resque/scheduler/delaying_extensions.rb +324 -0
  14. data/lib/resque/scheduler/env.rb +89 -0
  15. data/lib/resque/scheduler/extension.rb +13 -0
  16. data/lib/resque/scheduler/failure_handler.rb +11 -0
  17. data/lib/resque/scheduler/lock/base.rb +61 -0
  18. data/lib/resque/scheduler/lock/basic.rb +27 -0
  19. data/lib/resque/scheduler/lock/resilient.rb +78 -0
  20. data/lib/resque/scheduler/lock.rb +4 -0
  21. data/lib/resque/scheduler/locking.rb +104 -0
  22. data/lib/resque/scheduler/logger_builder.rb +72 -0
  23. data/lib/resque/scheduler/plugin.rb +31 -0
  24. data/lib/resque/scheduler/scheduling_extensions.rb +141 -0
  25. data/lib/resque/scheduler/server/views/delayed.erb +63 -0
  26. data/lib/resque/scheduler/server/views/delayed_schedules.erb +20 -0
  27. data/lib/resque/scheduler/server/views/delayed_timestamp.erb +26 -0
  28. data/lib/resque/scheduler/server/views/requeue-params.erb +23 -0
  29. data/lib/resque/scheduler/server/views/scheduler.erb +58 -0
  30. data/lib/resque/scheduler/server/views/search.erb +72 -0
  31. data/lib/resque/scheduler/server/views/search_form.erb +8 -0
  32. data/lib/resque/scheduler/server.rb +268 -0
  33. data/lib/resque/scheduler/signal_handling.rb +40 -0
  34. data/lib/resque/scheduler/tasks.rb +25 -0
  35. data/lib/resque/scheduler/util.rb +39 -0
  36. data/lib/resque/scheduler/version.rb +7 -0
  37. data/lib/resque/scheduler.rb +447 -0
  38. data/lib/resque-scheduler.rb +4 -0
  39. data/resque-scheduler.gemspec +60 -0
  40. metadata +330 -0
@@ -0,0 +1,447 @@
1
+ # vim:fileencoding=utf-8
2
+
3
+ require 'rufus/scheduler'
4
+ require_relative 'scheduler/configuration'
5
+ require_relative 'scheduler/locking'
6
+ require_relative 'scheduler/logger_builder'
7
+ require_relative 'scheduler/signal_handling'
8
+ require_relative 'scheduler/failure_handler'
9
+
10
+ module Resque
11
+ module Scheduler
12
+ autoload :Cli, 'resque/scheduler/cli'
13
+ autoload :Extension, 'resque/scheduler/extension'
14
+ autoload :Util, 'resque/scheduler/util'
15
+ autoload :VERSION, 'resque/scheduler/version'
16
+
17
+ private
18
+
19
+ extend Resque::Scheduler::Locking
20
+ extend Resque::Scheduler::Configuration
21
+ extend Resque::Scheduler::SignalHandling
22
+
23
+ public
24
+
25
+ class << self
26
+ attr_writer :logger
27
+
28
+ # the Rufus::Scheduler jobs that are scheduled
29
+ attr_reader :scheduled_jobs
30
+
31
+ # allow user to set an additional failure handler
32
+ attr_writer :failure_handler
33
+
34
+ # Schedule all jobs and continually look for delayed jobs (never returns)
35
+ def run
36
+ procline 'Starting'
37
+
38
+ # trap signals
39
+ register_signal_handlers
40
+
41
+ # Quote from the resque/worker.
42
+ # Fix buffering so we can `rake resque:scheduler > scheduler.log` and
43
+ # get output from the child in there.
44
+ $stdout.sync = true
45
+ $stderr.sync = true
46
+
47
+ # Load the schedule into rufus
48
+ # If dynamic is set, load that schedule otherwise use normal load
49
+ if dynamic
50
+ reload_schedule!
51
+ else
52
+ load_schedule!
53
+ end
54
+
55
+ begin
56
+ @th = Thread.current
57
+
58
+ # Now start the scheduling part of the loop.
59
+ loop do
60
+ begin
61
+ if master?
62
+ handle_delayed_items
63
+ update_schedule if dynamic
64
+ end
65
+ rescue Errno::EAGAIN, Errno::ECONNRESET, Redis::CannotConnectError => e
66
+ log! e.message
67
+ release_master_lock
68
+ end
69
+ poll_sleep
70
+ end
71
+
72
+ rescue Interrupt
73
+ log 'Exiting'
74
+ end
75
+ end
76
+
77
+ def print_schedule
78
+ if rufus_scheduler
79
+ log! "Scheduling Info\tLast Run"
80
+ scheduler_jobs = rufus_scheduler.jobs
81
+ scheduler_jobs.each do |_k, v|
82
+ log! "#{v.t}\t#{v.last}\t"
83
+ end
84
+ end
85
+ end
86
+
87
+ # Pulls the schedule from Resque.schedule and loads it into the
88
+ # rufus scheduler instance
89
+ def load_schedule!
90
+ procline 'Loading Schedule'
91
+
92
+ # Need to load the schedule from redis for the first time if dynamic
93
+ Resque.reload_schedule! if dynamic
94
+
95
+ log! 'Schedule empty! Set Resque.schedule' if Resque.schedule.empty?
96
+
97
+ @scheduled_jobs = {}
98
+
99
+ Resque.schedule.each do |name, config|
100
+ load_schedule_job(name, config)
101
+ end
102
+ Resque.redis.del(:schedules_changed)
103
+ procline 'Schedules Loaded'
104
+ end
105
+
106
+ # modify interval type value to value with options if options available
107
+ def optionizate_interval_value(value)
108
+ args = value
109
+ if args.is_a?(::Array)
110
+ return args.first if args.size > 2 || !args.last.is_a?(::Hash)
111
+ # symbolize keys of hash for options
112
+ args[2] = args[1].reduce({}) do |m, i|
113
+ key, value = i
114
+ m[(key.respond_to?(:to_sym) ? key.to_sym : key) || key] = value
115
+ m
116
+ end
117
+
118
+ args[2][:job] = true
119
+ args[1] = nil
120
+ end
121
+ args
122
+ end
123
+
124
+ # Loads a job schedule into the Rufus::Scheduler and stores it
125
+ # in @scheduled_jobs
126
+ def load_schedule_job(name, config)
127
+ # If `rails_env` or `env` is set in the config, load jobs only if they
128
+ # are meant to be loaded in `Resque::Scheduler.env`. If `rails_env` or
129
+ # `env` is missing, the job should be scheduled regardless of the value
130
+ # of `Resque::Scheduler.env`.
131
+
132
+ configured_env = config['rails_env'] || config['env']
133
+
134
+ if configured_env.nil? || env_matches?(configured_env)
135
+ log! "Scheduling #{name} "
136
+ interval_defined = false
137
+ interval_types = %w(cron every)
138
+ interval_types.each do |interval_type|
139
+ next unless !config[interval_type].nil? && !config[interval_type].empty?
140
+ args = optionizate_interval_value(config[interval_type])
141
+ args = [args, nil, job: true] if args.is_a?(::String)
142
+
143
+ job = rufus_scheduler.send(interval_type, *args) do
144
+ enqueue_recurring(name, config)
145
+ end
146
+ @scheduled_jobs[name] = job
147
+ interval_defined = true
148
+ break
149
+ end
150
+ unless interval_defined
151
+ log! "no #{interval_types.join(' / ')} found for " \
152
+ "#{config['class']} (#{name}) - skipping"
153
+ end
154
+ else
155
+ log "Skipping schedule of #{name} because configured " \
156
+ "env #{configured_env.inspect} does not match current " \
157
+ "env #{env.inspect}"
158
+ end
159
+ end
160
+
161
+ # Returns true if the given schedule config hash matches the current env
162
+ def rails_env_matches?(config)
163
+ warn '`Resque::Scheduler.rails_env_matches?` is deprecated. ' \
164
+ 'Please use `Resque::Scheduler.env_matches?` instead.'
165
+ config['rails_env'] && env &&
166
+ config['rails_env'].split(/[\s,]+/).include?(env)
167
+ end
168
+
169
+ # Returns true if the current env is non-nil and the configured env
170
+ # (which is a comma-split string) includes the current env.
171
+ def env_matches?(configured_env)
172
+ env && configured_env.split(/[\s,]+/).include?(env)
173
+ end
174
+
175
+ # Handles queueing delayed items
176
+ # at_time - Time to start scheduling items (default: now).
177
+ def handle_delayed_items(at_time = nil)
178
+ timestamp = Resque.next_delayed_timestamp(at_time)
179
+ if timestamp
180
+ procline 'Processing Delayed Items'
181
+ until timestamp.nil?
182
+ enqueue_delayed_items_for_timestamp(timestamp)
183
+ timestamp = Resque.next_delayed_timestamp(at_time)
184
+ end
185
+ end
186
+ end
187
+
188
+ def enqueue_next_item(timestamp)
189
+ item = Resque.next_item_for_timestamp(timestamp)
190
+
191
+ if item
192
+ log "queuing #{item['class']} [delayed]"
193
+ enqueue(item)
194
+ end
195
+
196
+ item
197
+ end
198
+
199
+ # Enqueues all delayed jobs for a timestamp
200
+ def enqueue_delayed_items_for_timestamp(timestamp)
201
+ item = nil
202
+ loop do
203
+ handle_shutdown do
204
+ # Continually check that it is still the master
205
+ item = enqueue_next_item(timestamp) if master?
206
+ end
207
+ # continue processing until there are no more ready items in this
208
+ # timestamp
209
+ break if item.nil?
210
+ end
211
+ end
212
+
213
+ def enqueue(config)
214
+ enqueue_from_config(config)
215
+ rescue => e
216
+ Resque::Scheduler.failure_handler.on_enqueue_failure(config, e)
217
+ end
218
+
219
+ def handle_shutdown
220
+ exit if @shutdown
221
+ yield
222
+ exit if @shutdown
223
+ end
224
+
225
+ # Enqueues a job based on a config hash
226
+ def enqueue_from_config(job_config)
227
+ args = job_config['args'] || job_config[:args]
228
+
229
+ klass_name = job_config['class'] || job_config[:class]
230
+ begin
231
+ klass = Resque::Scheduler::Util.constantize(klass_name)
232
+ rescue NameError
233
+ klass = klass_name
234
+ end
235
+
236
+ params = args.is_a?(Hash) ? [args] : Array(args)
237
+ queue = job_config['queue'] ||
238
+ job_config[:queue] ||
239
+ Resque.queue_from_class(klass)
240
+ # Support custom job classes like those that inherit from
241
+ # Resque::JobWithStatus (resque-status)
242
+ job_klass = job_config['custom_job_class']
243
+ if job_klass && job_klass != 'Resque::Job'
244
+ # The custom job class API must offer a static "scheduled" method. If
245
+ # the custom job class can not be constantized (via a requeue call
246
+ # from the web perhaps), fall back to enqueing normally via
247
+ # Resque::Job.create.
248
+ begin
249
+ Resque::Scheduler::Util.constantize(job_klass).scheduled(
250
+ queue, klass_name, *params
251
+ )
252
+ rescue NameError
253
+ # Note that the custom job class (job_config['custom_job_class'])
254
+ # is the one enqueued
255
+ Resque::Job.create(queue, job_klass, *params)
256
+ end
257
+ else
258
+ # Hack to avoid havoc for people shoving stuff into queues
259
+ # for non-existent classes (for example: running scheduler in
260
+ # one app that schedules for another.
261
+ if Class === klass
262
+ Resque::Scheduler::Plugin.run_before_delayed_enqueue_hooks(
263
+ klass, *params
264
+ )
265
+
266
+ # If the class is a custom job class, call self#scheduled on it.
267
+ # This allows you to do things like Resque.enqueue_at(timestamp,
268
+ # CustomJobClass). Otherwise, pass off to Resque.
269
+ if klass.respond_to?(:scheduled)
270
+ klass.scheduled(queue, klass_name, *params)
271
+ else
272
+ Resque.enqueue_to(queue, klass, *params)
273
+ end
274
+ else
275
+ # This will not run the before_hooks in rescue, but will at least
276
+ # queue the job.
277
+ Resque::Job.create(queue, klass, *params)
278
+ end
279
+ end
280
+ end
281
+
282
+ def rufus_scheduler
283
+ @rufus_scheduler ||= Rufus::Scheduler.new
284
+ end
285
+
286
+ # Stops old rufus scheduler and creates a new one. Returns the new
287
+ # rufus scheduler
288
+ def clear_schedule!
289
+ rufus_scheduler.stop
290
+ @rufus_scheduler = nil
291
+ @scheduled_jobs = {}
292
+ rufus_scheduler
293
+ end
294
+
295
+ def reload_schedule!
296
+ procline 'Reloading Schedule'
297
+ clear_schedule!
298
+ load_schedule!
299
+ end
300
+
301
+ def update_schedule
302
+ if Resque.redis.scard(:schedules_changed) > 0
303
+ procline 'Updating schedule'
304
+ loop do
305
+ schedule_name = Resque.redis.spop(:schedules_changed)
306
+ break unless schedule_name
307
+ Resque.reload_schedule!
308
+ if Resque.schedule.keys.include?(schedule_name)
309
+ unschedule_job(schedule_name)
310
+ load_schedule_job(schedule_name, Resque.schedule[schedule_name])
311
+ else
312
+ unschedule_job(schedule_name)
313
+ end
314
+ end
315
+ procline 'Schedules Loaded'
316
+ end
317
+ end
318
+
319
+ def unschedule_job(name)
320
+ if scheduled_jobs[name]
321
+ log "Removing schedule #{name}"
322
+ scheduled_jobs[name].unschedule
323
+ @scheduled_jobs.delete(name)
324
+ end
325
+ end
326
+
327
+ # Sleeps and returns true
328
+ def poll_sleep
329
+ handle_shutdown do
330
+ begin
331
+ poll_sleep_loop
332
+ ensure
333
+ @sleeping = false
334
+ end
335
+ end
336
+ true
337
+ end
338
+
339
+ def poll_sleep_loop
340
+ @sleeping = true
341
+ if poll_sleep_amount > 0
342
+ start = Time.now
343
+ loop do
344
+ elapsed_sleep = (Time.now - start)
345
+ remaining_sleep = poll_sleep_amount - elapsed_sleep
346
+ @do_break = false
347
+ if remaining_sleep <= 0
348
+ @do_break = true
349
+ else
350
+ @do_break = handle_signals_with_operation do
351
+ sleep(remaining_sleep)
352
+ end
353
+ end
354
+ break if @do_break
355
+ end
356
+ else
357
+ handle_signals_with_operation
358
+ end
359
+ end
360
+
361
+ def handle_signals_with_operation
362
+ yield if block_given?
363
+ handle_signals
364
+ false
365
+ rescue Interrupt
366
+ before_shutdown if @shutdown
367
+ true
368
+ end
369
+
370
+ def stop_rufus_scheduler
371
+ rufus_scheduler.shutdown(:wait)
372
+ rufus_scheduler.join
373
+ end
374
+
375
+ def before_shutdown
376
+ stop_rufus_scheduler
377
+ release_master_lock
378
+ end
379
+
380
+ # Sets the shutdown flag, clean schedules and exits if sleeping
381
+ def shutdown
382
+ return if @shutdown
383
+ @shutdown = true
384
+ log!('Shutting down')
385
+ @th.raise Interrupt if @sleeping
386
+ end
387
+
388
+ def log!(msg)
389
+ logger.info { msg }
390
+ end
391
+
392
+ def log_error(msg)
393
+ logger.error { msg }
394
+ end
395
+
396
+ def log(msg)
397
+ logger.debug { msg }
398
+ end
399
+
400
+ def procline(string)
401
+ log! string
402
+ argv0 = build_procline(string)
403
+ log "Setting procline #{argv0.inspect}"
404
+ $0 = argv0
405
+ end
406
+
407
+ def failure_handler
408
+ @failure_handler ||= Resque::Scheduler::FailureHandler
409
+ end
410
+
411
+ def logger
412
+ @logger ||= Resque::Scheduler::LoggerBuilder.new(
413
+ quiet: quiet,
414
+ verbose: verbose,
415
+ log_dev: logfile,
416
+ format: logformat
417
+ ).build
418
+ end
419
+
420
+ private
421
+
422
+ def enqueue_recurring(name, config)
423
+ if master?
424
+ log! "queueing #{config['class']} (#{name})"
425
+ Resque.last_enqueued_at(name, Time.now.to_s)
426
+ enqueue(config)
427
+ end
428
+ end
429
+
430
+ def app_str
431
+ app_name ? "[#{app_name}]" : ''
432
+ end
433
+
434
+ def env_str
435
+ env ? "[#{env}]" : ''
436
+ end
437
+
438
+ def build_procline(string)
439
+ "#{internal_name}#{app_str}#{env_str}: #{string}"
440
+ end
441
+
442
+ def internal_name
443
+ "resque-scheduler-#{Resque::Scheduler::VERSION}"
444
+ end
445
+ end
446
+ end
447
+ end
@@ -0,0 +1,4 @@
1
+ # vim:fileencoding=utf-8
2
+ require_relative 'resque/scheduler'
3
+
4
+ Resque.extend Resque::Scheduler::Extension
@@ -0,0 +1,60 @@
1
+ # vim:fileencoding=utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'resque/scheduler/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'resque-scheduler'
8
+ spec.version = Resque::Scheduler::VERSION
9
+ spec.authors = <<-EOF.split(/\n/).map(&:strip)
10
+ Ben VandenBos
11
+ Simon Eskildsen
12
+ Ryan Biesemeyer
13
+ Dan Buch
14
+ EOF
15
+ spec.email = %w(
16
+ bvandenbos@gmail.com
17
+ sirup@sirupsen.com
18
+ ryan@yaauie.com
19
+ dan@meatballhat.com
20
+ )
21
+ spec.summary = 'Light weight job scheduling on top of Resque'
22
+ spec.description = <<-DESCRIPTION
23
+ Light weight job scheduling on top of Resque.
24
+ Adds methods enqueue_at/enqueue_in to schedule jobs in the future.
25
+ Also supports queueing jobs on a fixed, cron-like schedule.
26
+ DESCRIPTION
27
+ spec.homepage = 'http://github.com/resque/resque-scheduler'
28
+ spec.license = 'MIT'
29
+
30
+ spec.files = `git ls-files -z`.split("\0").reject do |f|
31
+ f.match(%r{^(test|spec|features|examples|bin|tasks)/}) ||
32
+ f.match(/^(Vagrantfile|Gemfile\.lock|appveyor\.yml)/) ||
33
+ f.match(/^\.(rubocop|simplecov|travis|vagrant|gitignore)/)
34
+ end
35
+ spec.bindir = 'exe'
36
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
37
+ spec.require_paths = %w(lib)
38
+
39
+ spec.add_development_dependency 'bundler'
40
+ spec.add_development_dependency 'json'
41
+ spec.add_development_dependency 'kramdown'
42
+ spec.add_development_dependency 'minitest'
43
+ spec.add_development_dependency 'mocha'
44
+ spec.add_development_dependency 'pry'
45
+ spec.add_development_dependency 'rack-test'
46
+ spec.add_development_dependency 'rake'
47
+ spec.add_development_dependency 'simplecov'
48
+ spec.add_development_dependency 'test-unit'
49
+ spec.add_development_dependency 'yard'
50
+ spec.add_development_dependency 'tzinfo-data'
51
+
52
+ # We pin rubocop because new cops have a tendency to result in false-y
53
+ # positives for new contributors, which is not a nice experience.
54
+ spec.add_development_dependency 'rubocop', '~> 0.40.0'
55
+
56
+ spec.add_runtime_dependency 'mono_logger', '~> 1.0'
57
+ spec.add_runtime_dependency 'redis', '~> 3.3'
58
+ spec.add_runtime_dependency 'resque', '~> 1.26'
59
+ spec.add_runtime_dependency 'rufus-scheduler', '~> 3.2'
60
+ end