procrastinator 0.9.0 → 1.0.0.pre.rc3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.gitignore +6 -1
- data/.rubocop.yml +20 -1
- data/README.md +327 -333
- data/RELEASE_NOTES.md +44 -0
- data/lib/procrastinator/config.rb +93 -129
- data/lib/procrastinator/logged_task.rb +50 -0
- data/lib/procrastinator/queue.rb +168 -12
- data/lib/procrastinator/queue_worker.rb +52 -97
- data/lib/procrastinator/rake/daemon_tasks.rb +86 -0
- data/lib/procrastinator/rake/tasks.rb +3 -0
- data/lib/procrastinator/scheduler.rb +297 -78
- data/lib/procrastinator/task.rb +46 -28
- data/lib/procrastinator/task_meta_data.rb +96 -52
- data/lib/procrastinator/task_store/file_transaction.rb +76 -0
- data/lib/procrastinator/task_store/simple_comma_store.rb +164 -0
- data/lib/procrastinator/test/mocks.rb +35 -0
- data/lib/procrastinator/version.rb +1 -1
- data/lib/procrastinator.rb +9 -24
- data/procrastinator.gemspec +13 -9
- metadata +43 -26
- data/lib/procrastinator/loaders/csv_loader.rb +0 -107
- data/lib/procrastinator/queue_manager.rb +0 -201
- data/lib/procrastinator/task_worker.rb +0 -100
- data/lib/rake/procrastinator_task.rb +0 -34
@@ -1,5 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'stringio'
|
4
|
+
|
3
5
|
module Procrastinator
|
4
6
|
# A Scheduler object provides the API for client applications to manage delayed tasks.
|
5
7
|
#
|
@@ -7,33 +9,24 @@ module Procrastinator
|
|
7
9
|
#
|
8
10
|
# @author Robin Miller
|
9
11
|
class Scheduler
|
10
|
-
|
11
|
-
|
12
|
-
def_delegators :@queue_manager, :act
|
13
|
-
|
14
|
-
def initialize(config, queue_manager)
|
15
|
-
@config = config
|
16
|
-
@queue_manager = queue_manager
|
12
|
+
def initialize(config)
|
13
|
+
@config = config
|
17
14
|
end
|
18
15
|
|
19
16
|
# Records a new task to be executed at the given time.
|
20
17
|
#
|
21
|
-
# @param
|
18
|
+
# @param queue_name [Symbol] the symbol identifier for the queue to add a new task on
|
22
19
|
# @param run_at [Time, Integer] Optional time when this task should be executed. Defaults to the current time.
|
23
|
-
# @param data [Hash, Array] Optional simple data object to be provided to the task
|
20
|
+
# @param data [Hash, Array, String, Integer] Optional simple data object to be provided to the task on execution.
|
24
21
|
# @param expire_at [Time, Integer] Optional time when the task should be abandoned
|
25
|
-
def delay(
|
26
|
-
|
27
|
-
|
28
|
-
|
22
|
+
def delay(queue_name = nil, data: nil, run_at: Time.now, expire_at: nil)
|
23
|
+
raise ArgumentError, <<~ERR unless queue_name.nil? || queue_name.is_a?(Symbol)
|
24
|
+
must provide a queue name as the first argument. Received: #{ queue_name }
|
25
|
+
ERR
|
29
26
|
|
30
|
-
|
27
|
+
queue = @config.queue(name: queue_name)
|
31
28
|
|
32
|
-
|
33
|
-
run_at: run_at.to_i,
|
34
|
-
initial_run_at: run_at.to_i,
|
35
|
-
expire_at: expire_at.nil? ? nil : expire_at.to_i,
|
36
|
-
data: YAML.dump(data))
|
29
|
+
queue.create(run_at: run_at, expire_at: expire_at, data: data)
|
37
30
|
end
|
38
31
|
|
39
32
|
# Alters an existing task to run at a new time, expire at a new time, or both.
|
@@ -54,7 +47,7 @@ module Procrastinator
|
|
54
47
|
UpdateProxy.new(@config, identifier: identifier.merge(queue: queue.to_s))
|
55
48
|
end
|
56
49
|
|
57
|
-
# Removes an existing task, as located by the
|
50
|
+
# Removes an existing task, as located by the given identifying information.
|
58
51
|
#
|
59
52
|
# The identifier can include any data field stored in the task loader. Often this is the information in :data.
|
60
53
|
#
|
@@ -63,109 +56,335 @@ module Procrastinator
|
|
63
56
|
#
|
64
57
|
# @see TaskMetaData
|
65
58
|
def cancel(queue, identifier)
|
66
|
-
|
59
|
+
queue = @config.queue(name: queue)
|
60
|
+
|
61
|
+
tasks = queue.read(identifier.merge(queue: queue.name.to_s))
|
67
62
|
|
68
63
|
raise "no task matches search: #{ identifier }" if tasks.empty?
|
69
64
|
raise "multiple tasks match search: #{ identifier }" if tasks.size > 1
|
70
65
|
|
71
|
-
|
66
|
+
queue.delete(tasks.first[:id])
|
67
|
+
end
|
68
|
+
|
69
|
+
# Spawns a new worker thread for each queue defined in the config
|
70
|
+
#
|
71
|
+
# @param queue_names [Array<String,Symbol>] Names of specific queues to act upon.
|
72
|
+
# Omit or leave empty to act on all queues.
|
73
|
+
def work(*queue_names)
|
74
|
+
queue_names = @config.queues if queue_names.empty?
|
75
|
+
|
76
|
+
workers = queue_names.collect do |queue_name|
|
77
|
+
QueueWorker.new(queue: queue_name, config: @config)
|
78
|
+
end
|
79
|
+
|
80
|
+
WorkProxy.new(workers, @config)
|
72
81
|
end
|
73
82
|
|
74
83
|
# Provides a more natural syntax for rescheduling tasks
|
75
84
|
#
|
76
85
|
# @see Scheduler#reschedule
|
77
86
|
class UpdateProxy
|
78
|
-
def initialize(
|
79
|
-
|
80
|
-
|
81
|
-
@config = config
|
87
|
+
def initialize(queue, identifier:)
|
88
|
+
@queue = queue
|
82
89
|
@identifier = identifier
|
83
90
|
end
|
84
91
|
|
85
92
|
def to(run_at: nil, expire_at: nil)
|
86
|
-
task = fetch_task(@identifier)
|
87
|
-
|
88
|
-
verify_time_provided(run_at, expire_at)
|
89
|
-
validate_run_at(run_at, task[:expire_at], expire_at)
|
93
|
+
task = @queue.fetch_task(@identifier)
|
90
94
|
|
91
|
-
|
92
|
-
attempts: 0,
|
93
|
-
last_error: nil,
|
94
|
-
last_error_at: nil
|
95
|
-
}
|
95
|
+
raise ArgumentError, 'you must provide at least :run_at or :expire_at' if run_at.nil? && expire_at.nil?
|
96
96
|
|
97
|
-
|
98
|
-
|
97
|
+
task.reschedule(expire_at: expire_at) if expire_at
|
98
|
+
task.reschedule(run_at: run_at) if run_at
|
99
99
|
|
100
|
-
|
100
|
+
new_data = task.to_h
|
101
|
+
new_data.delete(:queue)
|
102
|
+
new_data.delete(:data)
|
103
|
+
@queue.update(new_data.delete(:id), new_data)
|
101
104
|
end
|
102
105
|
|
103
106
|
alias at to
|
107
|
+
end
|
108
|
+
|
109
|
+
# Serial work style
|
110
|
+
#
|
111
|
+
# @see WorkProxy
|
112
|
+
module SerialWorking
|
113
|
+
# Work off the given number of tasks for each queue and return
|
114
|
+
# @param steps [integer] The number of tasks to complete.
|
115
|
+
def serially(steps: 1)
|
116
|
+
steps.times do
|
117
|
+
workers.each(&:work_one)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
# Threaded work style
|
123
|
+
#
|
124
|
+
# @see WorkProxy
|
125
|
+
module ThreadedWorking
|
126
|
+
PROG_NAME = 'Procrastinator'
|
127
|
+
|
128
|
+
# Work off jobs per queue, each in its own thread.
|
129
|
+
#
|
130
|
+
# @param timeout Maximum number of seconds to run for. If nil, will run indefinitely.
|
131
|
+
def threaded(timeout: nil)
|
132
|
+
open_log
|
133
|
+
shutdown_on_interrupt
|
134
|
+
|
135
|
+
begin
|
136
|
+
@threads = spawn_threads
|
137
|
+
|
138
|
+
@logger.info "Procrastinator running. Process ID: #{ Process.pid }"
|
139
|
+
@threads.each do |thread|
|
140
|
+
thread.join(timeout)
|
141
|
+
end
|
142
|
+
rescue StandardError => e
|
143
|
+
thread_crash(e)
|
144
|
+
ensure
|
145
|
+
@logger&.info 'Halting worker threads...'
|
146
|
+
shutdown!
|
147
|
+
@logger&.info 'Threads halted.'
|
148
|
+
end
|
149
|
+
end
|
104
150
|
|
105
151
|
private
|
106
152
|
|
107
|
-
def
|
108
|
-
|
153
|
+
def spawn_threads
|
154
|
+
@logger.info "Starting workers for queues: #{ @workers.collect(&:name).join(', ') }"
|
155
|
+
|
156
|
+
@workers.collect do |worker|
|
157
|
+
@logger.debug "Spawning thread: #{ worker.name }"
|
158
|
+
Thread.new(worker) do |w|
|
159
|
+
Thread.current.abort_on_exception = true
|
160
|
+
Thread.current.thread_variable_set(:name, w.name)
|
161
|
+
|
162
|
+
begin
|
163
|
+
worker.work!
|
164
|
+
ensure
|
165
|
+
worker.halt
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
def thread_crash(error)
|
172
|
+
crashed_threads = (@threads || []).select { |t| t.status.nil? }.collect do |thread|
|
173
|
+
"Crashed thread: #{ thread.thread_variable_get(:name) }"
|
174
|
+
end
|
175
|
+
|
176
|
+
@logger.fatal <<~MSG
|
177
|
+
Crash detected in queue worker thread.
|
178
|
+
#{ crashed_threads.join("\n") }
|
179
|
+
#{ error.message }
|
180
|
+
#{ error.backtrace.join("\n\t") }"
|
181
|
+
MSG
|
109
182
|
end
|
110
183
|
|
111
|
-
def
|
112
|
-
|
184
|
+
def shutdown_on_interrupt
|
185
|
+
Signal.trap('INT') do
|
186
|
+
warn "\n" # just to separate the shutdown log item
|
187
|
+
shutdown!
|
188
|
+
end
|
189
|
+
end
|
190
|
+
|
191
|
+
def shutdown!
|
192
|
+
(@threads || []).select(&:alive?).each(&:kill)
|
193
|
+
end
|
194
|
+
|
195
|
+
def open_log(quiet: false)
|
196
|
+
return if @logger
|
113
197
|
|
114
|
-
|
198
|
+
log_devs = []
|
115
199
|
|
116
|
-
|
200
|
+
log_devs << StringIO.new if quiet && !@config.log_level
|
201
|
+
log_devs << $stderr unless quiet
|
202
|
+
log_devs << log_path.open('a') if @config.log_level
|
117
203
|
|
118
|
-
|
204
|
+
multi = MultiIO.new(*log_devs)
|
205
|
+
multi.sync = true
|
119
206
|
|
120
|
-
|
207
|
+
@logger = Logger.new(multi,
|
208
|
+
progname: PROG_NAME.downcase,
|
209
|
+
level: @config.log_level || Logger::INFO,
|
210
|
+
formatter: Config::DEFAULT_LOG_FORMATTER)
|
121
211
|
end
|
122
212
|
|
123
|
-
def
|
124
|
-
|
213
|
+
def log_path
|
214
|
+
path = @config.log_dir / "#{ PROG_NAME.downcase }.log"
|
215
|
+
path.dirname.mkpath
|
216
|
+
# FileUtils.touch(log_path)
|
217
|
+
path
|
218
|
+
end
|
125
219
|
|
126
|
-
|
127
|
-
|
220
|
+
# IO Multiplexer that forwards calls to a list of IO streams.
|
221
|
+
class MultiIO
|
222
|
+
def initialize(*stream)
|
223
|
+
@streams = stream
|
224
|
+
end
|
225
|
+
|
226
|
+
(IO.methods << :path << :sync=).uniq.each do |method_name|
|
227
|
+
define_method(method_name) do |*args|
|
228
|
+
able_streams(method_name).collect do |stream|
|
229
|
+
stream.send(method_name, *args)
|
230
|
+
end.last # forces consistent return result type for callers (but may lose some info)
|
231
|
+
end
|
232
|
+
end
|
128
233
|
|
129
|
-
|
234
|
+
private
|
235
|
+
|
236
|
+
def able_streams(method_name)
|
237
|
+
@streams.select { |stream| stream.respond_to?(method_name) }
|
238
|
+
end
|
130
239
|
end
|
131
240
|
end
|
132
241
|
|
133
|
-
|
242
|
+
# Daemonized work style
|
243
|
+
#
|
244
|
+
# @see WorkProxy
|
245
|
+
module DaemonWorking
|
246
|
+
PID_EXT = '.pid'
|
247
|
+
DEFAULT_PID_DIR = Pathname.new('/var/run/').freeze
|
248
|
+
|
249
|
+
# 15 chars is linux limit
|
250
|
+
MAX_PROC_LEN = 15
|
251
|
+
|
252
|
+
# Consumes the current process and turns it into a background daemon and proceed as #threaded.
|
253
|
+
# Additional logging is recorded in the directory specified by the Procrastinator.setup configuration.
|
254
|
+
#
|
255
|
+
# If pid_path ends with extension '.pid', the basename will be requested as process title (depending on OS
|
256
|
+
# support). An extensionless path is assumed to be a directory and a default filename (and proctitle) is used.
|
257
|
+
#
|
258
|
+
# @param pid_path [Pathname, File, String, nil] Path to where the process ID file is to be kept.
|
259
|
+
def daemonized!(pid_path = nil)
|
260
|
+
spawn_daemon(pid_path)
|
261
|
+
|
262
|
+
threaded
|
263
|
+
end
|
134
264
|
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
265
|
+
# Normalizes the given pid path, including conversion to absolute path and defaults.
|
266
|
+
#
|
267
|
+
# @param pid_path [Pathname, File, String, nil] path to normalize
|
268
|
+
def self.normalize_pid(pid_path)
|
269
|
+
normalized = Pathname.new(pid_path || DEFAULT_PID_DIR)
|
270
|
+
normalized /= "#{ PROG_NAME.downcase }#{ PID_EXT }" unless normalized.extname == PID_EXT
|
140
271
|
|
141
|
-
|
142
|
-
|
143
|
-
must provide a queue name as the first argument. Received: #{ queue_name }
|
144
|
-
ERR
|
272
|
+
normalized.expand_path
|
273
|
+
end
|
145
274
|
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
end
|
275
|
+
# Stops the procrastinator process denoted by the provided pid file
|
276
|
+
def self.halt!(pid_path)
|
277
|
+
pid_path = normalize_pid pid_path
|
150
278
|
|
151
|
-
|
152
|
-
|
279
|
+
Process.kill('TERM', pid_path.read.to_i)
|
280
|
+
end
|
281
|
+
|
282
|
+
def self.running?(pid_path)
|
283
|
+
pid = normalize_pid(pid_path).read.to_i
|
284
|
+
|
285
|
+
# this raises Errno::ESRCH when no process found, therefore if found we should exit
|
286
|
+
Process.getpgid pid
|
153
287
|
|
154
|
-
|
155
|
-
|
156
|
-
|
288
|
+
true
|
289
|
+
rescue Errno::ENOENT, Errno::ESRCH
|
290
|
+
false
|
157
291
|
end
|
158
292
|
|
159
|
-
|
160
|
-
|
161
|
-
|
293
|
+
private
|
294
|
+
|
295
|
+
def spawn_daemon(pid_path)
|
296
|
+
pid_path = DaemonWorking.normalize_pid pid_path
|
297
|
+
|
298
|
+
open_log quiet: true
|
299
|
+
@logger.info "Starting #{ PROG_NAME } daemon..."
|
300
|
+
|
301
|
+
print_debug_context
|
302
|
+
|
303
|
+
# "You, search from the spastic dentistry department down through disembowelment.
|
304
|
+
# You, cover children's dance recitals through holiday weekend IKEA. Go."
|
305
|
+
Process.daemon
|
306
|
+
|
307
|
+
manage_pid pid_path
|
308
|
+
rename_process pid_path
|
309
|
+
rescue StandardError => e
|
310
|
+
@logger&.fatal ([e.message] + e.backtrace).join("\n")
|
311
|
+
raise e
|
312
|
+
end
|
313
|
+
|
314
|
+
def manage_pid(pid_path)
|
315
|
+
ensure_unique(pid_path)
|
316
|
+
|
317
|
+
@logger.debug "Managing pid at path: #{ pid_path }"
|
318
|
+
pid_path.dirname.mkpath
|
319
|
+
pid_path.write Process.pid.to_s
|
320
|
+
|
321
|
+
at_exit do
|
322
|
+
if pid_path.exist?
|
323
|
+
@logger.debug "Cleaning up pid file #{ pid_path }"
|
324
|
+
pid_path.delete
|
325
|
+
end
|
326
|
+
@logger.info "Procrastinator (pid #{ Process.pid }) halted."
|
162
327
|
end
|
163
|
-
elsif !queue.task_class.method_defined?(:data=)
|
164
|
-
raise ArgumentError, <<~ERROR
|
165
|
-
task #{ queue.task_class } does not import :data. Add this in your class definition:
|
166
|
-
task_attr :data
|
167
|
-
ERROR
|
168
328
|
end
|
329
|
+
|
330
|
+
def ensure_unique(pid_path)
|
331
|
+
return unless pid_path.exist?
|
332
|
+
|
333
|
+
@logger.debug "Checking pid file #{ pid_path }"
|
334
|
+
|
335
|
+
if DaemonWorking.running? pid_path
|
336
|
+
hint = 'Either terminate that process or remove the pid file (if coincidental).'
|
337
|
+
msg = "Another process (pid #{ pid_path.read }) already exists for #{ pid_path }. #{ hint }"
|
338
|
+
@logger.fatal msg
|
339
|
+
raise ProcessExistsError, msg
|
340
|
+
else
|
341
|
+
@logger.warn "Replacing old pid file of defunct process (pid #{ pid_path.read }) at #{ pid_path }."
|
342
|
+
end
|
343
|
+
end
|
344
|
+
|
345
|
+
def print_debug_context
|
346
|
+
@logger.debug "Ruby Path: #{ ENV.fetch 'RUBY_ROOT' }"
|
347
|
+
@logger.debug "Bundler Path: #{ ENV.fetch 'BUNDLE_BIN_PATH' }"
|
348
|
+
# LOGNAME is the posix standard and is set by cron, so probably reliable.
|
349
|
+
@logger.debug "Runtime User: #{ ENV.fetch('LOGNAME') || ENV.fetch('USERNAME') }"
|
350
|
+
end
|
351
|
+
|
352
|
+
def rename_process(pid_path)
|
353
|
+
name = pid_path.basename(PID_EXT).to_s
|
354
|
+
|
355
|
+
if name.size > MAX_PROC_LEN
|
356
|
+
@logger.warn "Process name is longer than max length (#{ MAX_PROC_LEN }). Trimming to fit."
|
357
|
+
name = name[0, MAX_PROC_LEN]
|
358
|
+
end
|
359
|
+
|
360
|
+
if system('pidof', name, out: File::NULL)
|
361
|
+
@logger.warn "Another process is already named '#{ name }'. Consider the 'name:' keyword to distinguish."
|
362
|
+
end
|
363
|
+
|
364
|
+
@logger.debug "Renaming process to: #{ name }"
|
365
|
+
Process.setproctitle name
|
366
|
+
end
|
367
|
+
|
368
|
+
include ThreadedWorking
|
169
369
|
end
|
370
|
+
|
371
|
+
# DSL grammar object to enable chaining #work with the three work modes.
|
372
|
+
#
|
373
|
+
# @see Scheduler#work
|
374
|
+
class WorkProxy
|
375
|
+
include SerialWorking
|
376
|
+
include ThreadedWorking
|
377
|
+
include DaemonWorking
|
378
|
+
|
379
|
+
attr_reader :workers
|
380
|
+
|
381
|
+
def initialize(workers, config)
|
382
|
+
@workers = workers
|
383
|
+
@config = config
|
384
|
+
end
|
385
|
+
end
|
386
|
+
end
|
387
|
+
|
388
|
+
class ProcessExistsError < RuntimeError
|
170
389
|
end
|
171
390
|
end
|
data/lib/procrastinator/task.rb
CHANGED
@@ -1,46 +1,64 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'forwardable'
|
4
|
+
require 'time'
|
5
|
+
|
3
6
|
module Procrastinator
|
4
|
-
#
|
5
|
-
# for the task class to access additional information (data, logger, etc) from Procrastinator.
|
6
|
-
#
|
7
|
-
# If you are averse to including this in your task class, you can just declare an attr_accessor for the
|
8
|
-
# information you want Procrastinator to feed your task.
|
7
|
+
# Wraps a task handler and task metadata
|
9
8
|
#
|
10
9
|
# @author Robin Miller
|
11
|
-
|
12
|
-
|
10
|
+
class Task
|
11
|
+
extend Forwardable
|
13
12
|
|
14
|
-
|
15
|
-
|
16
|
-
|
13
|
+
def_delegators :@metadata,
|
14
|
+
:id, :run_at, :initial_run_at, :expire_at,
|
15
|
+
:attempts, :last_fail_at, :last_error,
|
16
|
+
:data, :to_h, :serialized_data,
|
17
|
+
:queue, :reschedule
|
17
18
|
|
18
|
-
def
|
19
|
-
|
19
|
+
def initialize(metadata, handler)
|
20
|
+
@metadata = metadata
|
21
|
+
@handler = handler
|
20
22
|
end
|
21
23
|
|
22
|
-
def
|
23
|
-
if
|
24
|
-
|
25
|
-
|
24
|
+
def run
|
25
|
+
raise ExpiredError, "task is over its expiry time of #{ @metadata.expire_at.iso8601 }" if @metadata.expired?
|
26
|
+
|
27
|
+
@metadata.add_attempt
|
28
|
+
result = Timeout.timeout(queue.timeout) do
|
29
|
+
@handler.run
|
26
30
|
end
|
31
|
+
@metadata.clear_fails
|
27
32
|
|
28
|
-
|
33
|
+
try_hook(:success, result)
|
29
34
|
end
|
30
35
|
|
31
|
-
|
32
|
-
module TaskClassMethods
|
33
|
-
def task_attr(*fields)
|
34
|
-
attr_list = KNOWN_ATTRIBUTES.collect { |a| ':' + a.to_s }.join(', ')
|
36
|
+
alias call run
|
35
37
|
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
38
|
+
# Records a failure in metadata and attempts to run the handler's #fail hook if present.
|
39
|
+
#
|
40
|
+
# @param error [StandardError] - the error that caused the failure
|
41
|
+
def fail(error)
|
42
|
+
hook = @metadata.failure(error)
|
41
43
|
|
42
|
-
|
43
|
-
|
44
|
+
try_hook(hook, error)
|
45
|
+
hook
|
46
|
+
end
|
47
|
+
|
48
|
+
def try_hook(method, *params)
|
49
|
+
@handler.send(method, *params) if @handler.respond_to? method
|
50
|
+
rescue StandardError => e
|
51
|
+
warn "#{ method.to_s.capitalize } hook error: #{ e.message }"
|
52
|
+
end
|
53
|
+
|
54
|
+
def to_s
|
55
|
+
"#{ @metadata.queue.name }##{ id } [#{ serialized_data }]"
|
56
|
+
end
|
57
|
+
|
58
|
+
class ExpiredError < RuntimeError
|
59
|
+
end
|
60
|
+
|
61
|
+
class AttemptsExhaustedError < RuntimeError
|
44
62
|
end
|
45
63
|
end
|
46
64
|
end
|