plines 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +12 -0
- data/LICENSE +22 -0
- data/README.md +420 -0
- data/Rakefile +61 -0
- data/lib/plines.rb +13 -0
- data/lib/plines/configuration.rb +55 -0
- data/lib/plines/dependency_graph.rb +81 -0
- data/lib/plines/dynamic_struct.rb +34 -0
- data/lib/plines/enqueued_job.rb +120 -0
- data/lib/plines/external_dependency_timeout.rb +30 -0
- data/lib/plines/indifferent_hash.rb +58 -0
- data/lib/plines/job.rb +88 -0
- data/lib/plines/job_batch.rb +363 -0
- data/lib/plines/job_batch_list.rb +57 -0
- data/lib/plines/job_enqueuer.rb +83 -0
- data/lib/plines/pipeline.rb +97 -0
- data/lib/plines/redis_objects.rb +108 -0
- data/lib/plines/step.rb +269 -0
- data/lib/plines/version.rb +3 -0
- metadata +192 -0
data/lib/plines/job.rb
ADDED
@@ -0,0 +1,88 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
require 'plines/indifferent_hash'
|
3
|
+
|
4
|
+
module Plines
|
5
|
+
# An instance of a Step: a step class paired with some data for the job.
|
6
|
+
Job = Struct.new(:klass, :data) do
|
7
|
+
extend Forwardable
|
8
|
+
attr_reader :dependencies, :dependents
|
9
|
+
def_delegators :klass, :qless_queue, :processing_queue
|
10
|
+
|
11
|
+
def initialize(klass, data)
|
12
|
+
super(klass, IndifferentHash.from(data))
|
13
|
+
@dependencies = Set.new
|
14
|
+
@dependents = Set.new
|
15
|
+
yield self if block_given?
|
16
|
+
end
|
17
|
+
|
18
|
+
def add_dependency(step)
|
19
|
+
dependencies << step
|
20
|
+
step.dependents << self
|
21
|
+
end
|
22
|
+
|
23
|
+
RemoveNonexistentDependencyError = Class.new(StandardError)
|
24
|
+
def remove_dependency(step)
|
25
|
+
unless dependencies.delete?(step) && step.dependents.delete?(self)
|
26
|
+
raise RemoveNonexistentDependencyError,
|
27
|
+
"Attempted to remove nonexistent dependency #{step} from #{self}"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def add_dependencies_for(batch_data)
|
32
|
+
klass.dependencies_for(self, batch_data).each do |job|
|
33
|
+
add_dependency(job)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def external_dependencies
|
38
|
+
klass.external_dependencies_for(data)
|
39
|
+
end
|
40
|
+
|
41
|
+
class << self
|
42
|
+
# Prevent users of this class from constructing a new instance directly;
|
43
|
+
# Instead, they should use #build.
|
44
|
+
#
|
45
|
+
# Note: I tried to override #new (w/ a `super` call) but it didn't work...
|
46
|
+
# I think it was overriding Struct.new rather than Job.new
|
47
|
+
# or something.
|
48
|
+
private :new
|
49
|
+
|
50
|
+
# Ensures all "identical" instances (same klass and data)
|
51
|
+
# created within the block are in fact the same object.
|
52
|
+
# This is important when constructing the dependency graph,
|
53
|
+
# so that all the dependency/dependee relationships point to
|
54
|
+
# the right objects (rather than duplicate objects).
|
55
|
+
def accumulate_instances
|
56
|
+
self.repository = Hash.new { |h,k| h[k] = new(*k) }
|
57
|
+
|
58
|
+
begin
|
59
|
+
yield
|
60
|
+
return repository.values
|
61
|
+
ensure
|
62
|
+
self.repository = nil
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def build(*args, &block)
|
67
|
+
repository[args, &block]
|
68
|
+
end
|
69
|
+
|
70
|
+
private
|
71
|
+
|
72
|
+
def repository=(value)
|
73
|
+
Thread.current[:plines_job_repository] = value
|
74
|
+
end
|
75
|
+
|
76
|
+
NullRepository = Class.new do
|
77
|
+
def self.[](args, &block)
|
78
|
+
Job.send(:new, *args, &block)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def repository
|
83
|
+
Thread.current[:plines_job_repository] || NullRepository
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
@@ -0,0 +1,363 @@
|
|
1
|
+
require 'time'
|
2
|
+
require 'json'
|
3
|
+
require 'plines/redis_objects'
|
4
|
+
require 'plines/indifferent_hash'
|
5
|
+
|
6
|
+
module Plines
|
7
|
+
# Represents a group of jobs that are enqueued together as a batch,
|
8
|
+
# based on the step dependency graph.
|
9
|
+
class JobBatch < Struct.new(:pipeline, :id)
|
10
|
+
include Plines::RedisObjectsHelpers
|
11
|
+
|
12
|
+
JobNotPendingError = Class.new(ArgumentError)
|
13
|
+
|
14
|
+
set :pending_job_jids
|
15
|
+
set :completed_job_jids
|
16
|
+
set :timed_out_external_deps
|
17
|
+
|
18
|
+
hash_key :meta
|
19
|
+
attr_reader :qless, :redis
|
20
|
+
|
21
|
+
def initialize(qless, pipeline, id)
|
22
|
+
@qless = qless
|
23
|
+
@redis = qless.redis
|
24
|
+
@allowed_to_add_external_deps = false
|
25
|
+
super(pipeline, id)
|
26
|
+
yield self if block_given?
|
27
|
+
end
|
28
|
+
|
29
|
+
BATCH_DATA_KEY = "batch_data"
|
30
|
+
EXT_DEP_KEYS_KEY = "ext_dep_keys"
|
31
|
+
|
32
|
+
# We use find/create in place of new for both
|
33
|
+
# so that the semantics of the two cases are clear.
|
34
|
+
private_class_method :new
|
35
|
+
|
36
|
+
CannotFindExistingJobBatchError = Class.new(StandardError)
|
37
|
+
|
38
|
+
def self.find(qless, pipeline, id)
|
39
|
+
new(qless, pipeline, id) do |inst|
|
40
|
+
unless inst.created_at
|
41
|
+
raise CannotFindExistingJobBatchError,
|
42
|
+
"Cannot find an existing job batch for #{pipeline} / #{id}"
|
43
|
+
end
|
44
|
+
|
45
|
+
yield inst if block_given?
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
JobBatchAlreadyCreatedError = Class.new(StandardError)
|
50
|
+
AddingExternalDependencyNotAllowedError = Class.new(StandardError)
|
51
|
+
|
52
|
+
def self.create(qless, pipeline, id, batch_data, options = {})
|
53
|
+
new(qless, pipeline, id) do |inst|
|
54
|
+
if inst.created_at
|
55
|
+
raise JobBatchAlreadyCreatedError,
|
56
|
+
"Job batch #{pipeline} / #{id} already exists"
|
57
|
+
end
|
58
|
+
|
59
|
+
inst.send(:populate_meta_for_create, batch_data, options)
|
60
|
+
|
61
|
+
inst.populate_external_deps_meta { yield inst if block_given? }
|
62
|
+
inst.meta.delete(:creation_in_progress)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def populate_external_deps_meta
|
67
|
+
@allowed_to_add_external_deps = true
|
68
|
+
yield
|
69
|
+
ext_deps = external_deps | newly_added_external_deps.to_a
|
70
|
+
meta[EXT_DEP_KEYS_KEY] = JSON.dump(ext_deps)
|
71
|
+
ensure
|
72
|
+
@allowed_to_add_external_deps = false
|
73
|
+
end
|
74
|
+
|
75
|
+
def newly_added_external_deps
|
76
|
+
@newly_added_external_deps ||= []
|
77
|
+
end
|
78
|
+
|
79
|
+
def external_deps
|
80
|
+
if keys = meta[EXT_DEP_KEYS_KEY]
|
81
|
+
decode(keys)
|
82
|
+
else
|
83
|
+
[]
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
def add_job(jid, *external_dependencies)
|
88
|
+
pending_job_jids << jid
|
89
|
+
|
90
|
+
unless @allowed_to_add_external_deps || external_dependencies.none?
|
91
|
+
raise AddingExternalDependencyNotAllowedError, "You cannot add jobs " +
|
92
|
+
"with external dependencies after creating the job batch."
|
93
|
+
else
|
94
|
+
external_dependencies.each do |dep|
|
95
|
+
newly_added_external_deps << dep
|
96
|
+
external_dependency_sets[dep] << jid
|
97
|
+
end
|
98
|
+
|
99
|
+
EnqueuedJob.create(qless, pipeline, jid, *external_dependencies)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
def job_jids
|
104
|
+
pending_job_jids | completed_job_jids
|
105
|
+
end
|
106
|
+
|
107
|
+
def jobs
|
108
|
+
job_jids.map { |jid| EnqueuedJob.new(qless, pipeline, jid) }
|
109
|
+
end
|
110
|
+
|
111
|
+
def job_repository
|
112
|
+
qless.jobs
|
113
|
+
end
|
114
|
+
|
115
|
+
def pending_qless_jobs
|
116
|
+
pending_job_jids.map do |jid|
|
117
|
+
job_repository[jid]
|
118
|
+
end.compact
|
119
|
+
end
|
120
|
+
|
121
|
+
def qless_jobs
|
122
|
+
job_jids.map do |jid|
|
123
|
+
job_repository[jid]
|
124
|
+
end.compact
|
125
|
+
end
|
126
|
+
|
127
|
+
def mark_job_as_complete(jid)
|
128
|
+
moved, pending_count, complete_count = redis.multi do
|
129
|
+
pending_job_jids.move(jid, completed_job_jids)
|
130
|
+
pending_job_jids.length
|
131
|
+
completed_job_jids.length
|
132
|
+
end
|
133
|
+
|
134
|
+
unless moved
|
135
|
+
raise JobNotPendingError,
|
136
|
+
"Jid #{jid} cannot be marked as complete for " +
|
137
|
+
"job batch #{id} since it is not pending"
|
138
|
+
end
|
139
|
+
|
140
|
+
if _complete?(pending_count, complete_count)
|
141
|
+
meta["completed_at"] = Time.now.iso8601
|
142
|
+
set_expiration!
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
def complete?
|
147
|
+
_complete?(pending_job_jids.length, completed_job_jids.length)
|
148
|
+
end
|
149
|
+
|
150
|
+
def resolve_external_dependency(dep_name)
|
151
|
+
jids = external_dependency_sets[dep_name]
|
152
|
+
|
153
|
+
update_external_dependency \
|
154
|
+
dep_name, :resolve_external_dependency, jids
|
155
|
+
|
156
|
+
cancel_timeout_job_jid_set_for(dep_name)
|
157
|
+
end
|
158
|
+
|
159
|
+
def timeout_external_dependency(dep_name, jids)
|
160
|
+
update_external_dependency \
|
161
|
+
dep_name, :timeout_external_dependency, Array(jids)
|
162
|
+
|
163
|
+
timed_out_external_deps << dep_name
|
164
|
+
end
|
165
|
+
|
166
|
+
def has_unresolved_external_dependency?(dep_name)
|
167
|
+
external_dependency_sets[dep_name].any? do |jid|
|
168
|
+
EnqueuedJob.new(qless, pipeline, jid)
|
169
|
+
.unresolved_external_dependencies.include?(dep_name)
|
170
|
+
end
|
171
|
+
end
|
172
|
+
|
173
|
+
def timed_out_external_dependencies
|
174
|
+
timed_out_external_deps.to_a
|
175
|
+
end
|
176
|
+
|
177
|
+
def created_at
|
178
|
+
time_from "created_at"
|
179
|
+
end
|
180
|
+
|
181
|
+
def completed_at
|
182
|
+
time_from "completed_at"
|
183
|
+
end
|
184
|
+
|
185
|
+
def cancelled?
|
186
|
+
meta["cancelled"] == "1"
|
187
|
+
end
|
188
|
+
|
189
|
+
def creation_in_progress?
|
190
|
+
meta["creation_in_progress"] == "1"
|
191
|
+
end
|
192
|
+
|
193
|
+
def timeout_reduction
|
194
|
+
@timeout_reduction ||= meta["timeout_reduction"].to_i
|
195
|
+
end
|
196
|
+
|
197
|
+
def spawned_from
|
198
|
+
return @spawned_from if defined?(@spawned_from)
|
199
|
+
|
200
|
+
if id = meta["spawned_from_id"]
|
201
|
+
@spawned_from = self.class.find(qless, pipeline, id)
|
202
|
+
else
|
203
|
+
@spawned_from = nil
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
CannotCancelError = Class.new(StandardError)
|
208
|
+
|
209
|
+
def cancel!
|
210
|
+
if complete?
|
211
|
+
raise CannotCancelError,
|
212
|
+
"JobBatch #{id} is already complete and cannot be cancelled"
|
213
|
+
end
|
214
|
+
|
215
|
+
perform_cancellation
|
216
|
+
end
|
217
|
+
|
218
|
+
def cancel
|
219
|
+
return false if complete?
|
220
|
+
perform_cancellation
|
221
|
+
end
|
222
|
+
|
223
|
+
def data
|
224
|
+
data = decode(meta[BATCH_DATA_KEY])
|
225
|
+
data && IndifferentHash.from(data)
|
226
|
+
end
|
227
|
+
|
228
|
+
def track_timeout_job(dep_name, jid)
|
229
|
+
timeout_job_jid_sets[dep_name] << jid
|
230
|
+
end
|
231
|
+
|
232
|
+
def timeout_job_jid_sets
|
233
|
+
@timeout_job_jid_sets ||= Hash.new do |hash, dep|
|
234
|
+
key = [key_prefix, "timeout_job_jids", dep].join(':')
|
235
|
+
hash[dep] = Redis::Set.new(key, redis)
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
239
|
+
SpawnOptions = Struct.new(:data_overrides, :timeout_reduction)
|
240
|
+
|
241
|
+
def spawn_copy
|
242
|
+
options = SpawnOptions.new({})
|
243
|
+
yield options if block_given?
|
244
|
+
overrides = JSON.parse(JSON.dump options.data_overrides)
|
245
|
+
|
246
|
+
pipeline.enqueue_jobs_for(data.merge(overrides), {
|
247
|
+
spawned_from_id: id,
|
248
|
+
timeout_reduction: options.timeout_reduction || 0
|
249
|
+
})
|
250
|
+
end
|
251
|
+
|
252
|
+
private
|
253
|
+
|
254
|
+
def populate_meta_for_create(batch_data, options)
|
255
|
+
metadata = {
|
256
|
+
created_at: Time.now.getutc.iso8601,
|
257
|
+
timeout_reduction: 0,
|
258
|
+
BATCH_DATA_KEY => JSON.dump(batch_data),
|
259
|
+
creation_in_progress: 1
|
260
|
+
}.merge(options)
|
261
|
+
|
262
|
+
meta.fill(metadata)
|
263
|
+
@timeout_reduction = metadata.fetch(:timeout_reduction)
|
264
|
+
end
|
265
|
+
|
266
|
+
SomeJobsFailedToCancelError = Class.new(StandardError)
|
267
|
+
CreationInStillInProgressError = Class.new(StandardError)
|
268
|
+
|
269
|
+
def perform_cancellation
|
270
|
+
return true if cancelled?
|
271
|
+
|
272
|
+
if creation_in_progress?
|
273
|
+
raise CreationInStillInProgressError,
|
274
|
+
"#{id} is still being created (started " +
|
275
|
+
"#{Time.now - created_at} seconds ago)"
|
276
|
+
end
|
277
|
+
|
278
|
+
qless.bulk_cancel(job_jids)
|
279
|
+
verify_all_jobs_cancelled
|
280
|
+
|
281
|
+
external_deps.each do |key|
|
282
|
+
cancel_timeout_job_jid_set_for(key)
|
283
|
+
end
|
284
|
+
|
285
|
+
meta["cancelled"] = "1"
|
286
|
+
set_expiration!
|
287
|
+
pipeline.configuration.notify(:after_job_batch_cancellation, self)
|
288
|
+
end
|
289
|
+
|
290
|
+
def verify_all_jobs_cancelled
|
291
|
+
jobs = qless_jobs.reject { |j| j.state == "complete" }
|
292
|
+
return if jobs.none?
|
293
|
+
|
294
|
+
raise SomeJobsFailedToCancelError,
|
295
|
+
"#{jobs.size} jobs failed to cancel: #{jobs.inspect}"
|
296
|
+
end
|
297
|
+
|
298
|
+
def update_external_dependency(dep_name, meth, jids)
|
299
|
+
jids.each do |jid|
|
300
|
+
EnqueuedJob.new(qless, pipeline, jid).send(meth, dep_name)
|
301
|
+
end
|
302
|
+
end
|
303
|
+
|
304
|
+
def _complete?(pending_size, complete_size)
|
305
|
+
pending_size == 0 && complete_size > 0
|
306
|
+
end
|
307
|
+
|
308
|
+
def time_from(meta_entry)
|
309
|
+
date_string = meta[meta_entry]
|
310
|
+
Time.iso8601(date_string) if date_string
|
311
|
+
end
|
312
|
+
|
313
|
+
def set_expiration!
|
314
|
+
keys_to_expire = declared_redis_object_keys.to_set
|
315
|
+
|
316
|
+
each_enqueued_job do |job|
|
317
|
+
keys_to_expire.merge(job.declared_redis_object_keys)
|
318
|
+
|
319
|
+
job.all_external_dependencies.each do |dep|
|
320
|
+
keys_to_expire << external_dependency_sets[dep].key
|
321
|
+
keys_to_expire << timeout_job_jid_sets[dep].key
|
322
|
+
end
|
323
|
+
end
|
324
|
+
|
325
|
+
set_expiration_on(*keys_to_expire)
|
326
|
+
end
|
327
|
+
|
328
|
+
def each_enqueued_job
|
329
|
+
job_jids.each do |jid|
|
330
|
+
yield EnqueuedJob.new(qless, pipeline, jid)
|
331
|
+
end
|
332
|
+
end
|
333
|
+
|
334
|
+
def external_dependency_sets
|
335
|
+
@external_dependency_sets ||= Hash.new do |hash, dep|
|
336
|
+
key = [key_prefix, "ext_deps", dep].join(':')
|
337
|
+
hash[dep] = Redis::Set.new(key, redis)
|
338
|
+
end
|
339
|
+
end
|
340
|
+
|
341
|
+
def decode(string)
|
342
|
+
string && JSON.load(string)
|
343
|
+
end
|
344
|
+
|
345
|
+
def cancel_timeout_job_jid_set_for(dep_name)
|
346
|
+
timeout_job_jid_set = timeout_job_jid_sets[dep_name]
|
347
|
+
timeout_job_jid_set.each { |jid| gracefully_cancel(jid) }
|
348
|
+
timeout_job_jid_set.del
|
349
|
+
end
|
350
|
+
|
351
|
+
def gracefully_cancel(jid)
|
352
|
+
job = job_repository[jid]
|
353
|
+
job && job.cancel
|
354
|
+
end
|
355
|
+
|
356
|
+
def set_expiration_on(*redis_keys)
|
357
|
+
redis_keys.each do |key|
|
358
|
+
redis.pexpire(key, pipeline.configuration.data_ttl_in_milliseconds)
|
359
|
+
end
|
360
|
+
end
|
361
|
+
end
|
362
|
+
end
|
363
|
+
|