plines 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,88 @@
1
+ require 'forwardable'
2
+ require 'plines/indifferent_hash'
3
+
4
+ module Plines
5
+ # An instance of a Step: a step class paired with some data for the job.
6
+ Job = Struct.new(:klass, :data) do
7
+ extend Forwardable
8
+ attr_reader :dependencies, :dependents
9
+ def_delegators :klass, :qless_queue, :processing_queue
10
+
11
+ def initialize(klass, data)
12
+ super(klass, IndifferentHash.from(data))
13
+ @dependencies = Set.new
14
+ @dependents = Set.new
15
+ yield self if block_given?
16
+ end
17
+
18
+ def add_dependency(step)
19
+ dependencies << step
20
+ step.dependents << self
21
+ end
22
+
23
+ RemoveNonexistentDependencyError = Class.new(StandardError)
24
+ def remove_dependency(step)
25
+ unless dependencies.delete?(step) && step.dependents.delete?(self)
26
+ raise RemoveNonexistentDependencyError,
27
+ "Attempted to remove nonexistent dependency #{step} from #{self}"
28
+ end
29
+ end
30
+
31
+ def add_dependencies_for(batch_data)
32
+ klass.dependencies_for(self, batch_data).each do |job|
33
+ add_dependency(job)
34
+ end
35
+ end
36
+
37
+ def external_dependencies
38
+ klass.external_dependencies_for(data)
39
+ end
40
+
41
+ class << self
42
+ # Prevent users of this class from constructing a new instance directly;
43
+ # Instead, they should use #build.
44
+ #
45
+ # Note: I tried to override #new (w/ a `super` call) but it didn't work...
46
+ # I think it was overriding Struct.new rather than Job.new
47
+ # or something.
48
+ private :new
49
+
50
+ # Ensures all "identical" instances (same klass and data)
51
+ # created within the block are in fact the same object.
52
+ # This is important when constructing the dependency graph,
53
+ # so that all the dependency/dependee relationships point to
54
+ # the right objects (rather than duplicate objects).
55
+ def accumulate_instances
56
+ self.repository = Hash.new { |h,k| h[k] = new(*k) }
57
+
58
+ begin
59
+ yield
60
+ return repository.values
61
+ ensure
62
+ self.repository = nil
63
+ end
64
+ end
65
+
66
+ def build(*args, &block)
67
+ repository[args, &block]
68
+ end
69
+
70
+ private
71
+
72
+ def repository=(value)
73
+ Thread.current[:plines_job_repository] = value
74
+ end
75
+
76
+ NullRepository = Class.new do
77
+ def self.[](args, &block)
78
+ Job.send(:new, *args, &block)
79
+ end
80
+ end
81
+
82
+ def repository
83
+ Thread.current[:plines_job_repository] || NullRepository
84
+ end
85
+ end
86
+ end
87
+ end
88
+
@@ -0,0 +1,363 @@
1
+ require 'time'
2
+ require 'json'
3
+ require 'plines/redis_objects'
4
+ require 'plines/indifferent_hash'
5
+
6
+ module Plines
7
+ # Represents a group of jobs that are enqueued together as a batch,
8
+ # based on the step dependency graph.
9
+ class JobBatch < Struct.new(:pipeline, :id)
10
+ include Plines::RedisObjectsHelpers
11
+
12
+ JobNotPendingError = Class.new(ArgumentError)
13
+
14
+ set :pending_job_jids
15
+ set :completed_job_jids
16
+ set :timed_out_external_deps
17
+
18
+ hash_key :meta
19
+ attr_reader :qless, :redis
20
+
21
+ def initialize(qless, pipeline, id)
22
+ @qless = qless
23
+ @redis = qless.redis
24
+ @allowed_to_add_external_deps = false
25
+ super(pipeline, id)
26
+ yield self if block_given?
27
+ end
28
+
29
+ BATCH_DATA_KEY = "batch_data"
30
+ EXT_DEP_KEYS_KEY = "ext_dep_keys"
31
+
32
+ # We use find/create in place of new for both
33
+ # so that the semantics of the two cases are clear.
34
+ private_class_method :new
35
+
36
+ CannotFindExistingJobBatchError = Class.new(StandardError)
37
+
38
+ def self.find(qless, pipeline, id)
39
+ new(qless, pipeline, id) do |inst|
40
+ unless inst.created_at
41
+ raise CannotFindExistingJobBatchError,
42
+ "Cannot find an existing job batch for #{pipeline} / #{id}"
43
+ end
44
+
45
+ yield inst if block_given?
46
+ end
47
+ end
48
+
49
+ JobBatchAlreadyCreatedError = Class.new(StandardError)
50
+ AddingExternalDependencyNotAllowedError = Class.new(StandardError)
51
+
52
+ def self.create(qless, pipeline, id, batch_data, options = {})
53
+ new(qless, pipeline, id) do |inst|
54
+ if inst.created_at
55
+ raise JobBatchAlreadyCreatedError,
56
+ "Job batch #{pipeline} / #{id} already exists"
57
+ end
58
+
59
+ inst.send(:populate_meta_for_create, batch_data, options)
60
+
61
+ inst.populate_external_deps_meta { yield inst if block_given? }
62
+ inst.meta.delete(:creation_in_progress)
63
+ end
64
+ end
65
+
66
+ def populate_external_deps_meta
67
+ @allowed_to_add_external_deps = true
68
+ yield
69
+ ext_deps = external_deps | newly_added_external_deps.to_a
70
+ meta[EXT_DEP_KEYS_KEY] = JSON.dump(ext_deps)
71
+ ensure
72
+ @allowed_to_add_external_deps = false
73
+ end
74
+
75
+ def newly_added_external_deps
76
+ @newly_added_external_deps ||= []
77
+ end
78
+
79
+ def external_deps
80
+ if keys = meta[EXT_DEP_KEYS_KEY]
81
+ decode(keys)
82
+ else
83
+ []
84
+ end
85
+ end
86
+
87
+ def add_job(jid, *external_dependencies)
88
+ pending_job_jids << jid
89
+
90
+ unless @allowed_to_add_external_deps || external_dependencies.none?
91
+ raise AddingExternalDependencyNotAllowedError, "You cannot add jobs " +
92
+ "with external dependencies after creating the job batch."
93
+ else
94
+ external_dependencies.each do |dep|
95
+ newly_added_external_deps << dep
96
+ external_dependency_sets[dep] << jid
97
+ end
98
+
99
+ EnqueuedJob.create(qless, pipeline, jid, *external_dependencies)
100
+ end
101
+ end
102
+
103
+ def job_jids
104
+ pending_job_jids | completed_job_jids
105
+ end
106
+
107
+ def jobs
108
+ job_jids.map { |jid| EnqueuedJob.new(qless, pipeline, jid) }
109
+ end
110
+
111
+ def job_repository
112
+ qless.jobs
113
+ end
114
+
115
+ def pending_qless_jobs
116
+ pending_job_jids.map do |jid|
117
+ job_repository[jid]
118
+ end.compact
119
+ end
120
+
121
+ def qless_jobs
122
+ job_jids.map do |jid|
123
+ job_repository[jid]
124
+ end.compact
125
+ end
126
+
127
+ def mark_job_as_complete(jid)
128
+ moved, pending_count, complete_count = redis.multi do
129
+ pending_job_jids.move(jid, completed_job_jids)
130
+ pending_job_jids.length
131
+ completed_job_jids.length
132
+ end
133
+
134
+ unless moved
135
+ raise JobNotPendingError,
136
+ "Jid #{jid} cannot be marked as complete for " +
137
+ "job batch #{id} since it is not pending"
138
+ end
139
+
140
+ if _complete?(pending_count, complete_count)
141
+ meta["completed_at"] = Time.now.iso8601
142
+ set_expiration!
143
+ end
144
+ end
145
+
146
+ def complete?
147
+ _complete?(pending_job_jids.length, completed_job_jids.length)
148
+ end
149
+
150
+ def resolve_external_dependency(dep_name)
151
+ jids = external_dependency_sets[dep_name]
152
+
153
+ update_external_dependency \
154
+ dep_name, :resolve_external_dependency, jids
155
+
156
+ cancel_timeout_job_jid_set_for(dep_name)
157
+ end
158
+
159
+ def timeout_external_dependency(dep_name, jids)
160
+ update_external_dependency \
161
+ dep_name, :timeout_external_dependency, Array(jids)
162
+
163
+ timed_out_external_deps << dep_name
164
+ end
165
+
166
+ def has_unresolved_external_dependency?(dep_name)
167
+ external_dependency_sets[dep_name].any? do |jid|
168
+ EnqueuedJob.new(qless, pipeline, jid)
169
+ .unresolved_external_dependencies.include?(dep_name)
170
+ end
171
+ end
172
+
173
+ def timed_out_external_dependencies
174
+ timed_out_external_deps.to_a
175
+ end
176
+
177
+ def created_at
178
+ time_from "created_at"
179
+ end
180
+
181
+ def completed_at
182
+ time_from "completed_at"
183
+ end
184
+
185
+ def cancelled?
186
+ meta["cancelled"] == "1"
187
+ end
188
+
189
+ def creation_in_progress?
190
+ meta["creation_in_progress"] == "1"
191
+ end
192
+
193
+ def timeout_reduction
194
+ @timeout_reduction ||= meta["timeout_reduction"].to_i
195
+ end
196
+
197
+ def spawned_from
198
+ return @spawned_from if defined?(@spawned_from)
199
+
200
+ if id = meta["spawned_from_id"]
201
+ @spawned_from = self.class.find(qless, pipeline, id)
202
+ else
203
+ @spawned_from = nil
204
+ end
205
+ end
206
+
207
+ CannotCancelError = Class.new(StandardError)
208
+
209
+ def cancel!
210
+ if complete?
211
+ raise CannotCancelError,
212
+ "JobBatch #{id} is already complete and cannot be cancelled"
213
+ end
214
+
215
+ perform_cancellation
216
+ end
217
+
218
+ def cancel
219
+ return false if complete?
220
+ perform_cancellation
221
+ end
222
+
223
+ def data
224
+ data = decode(meta[BATCH_DATA_KEY])
225
+ data && IndifferentHash.from(data)
226
+ end
227
+
228
+ def track_timeout_job(dep_name, jid)
229
+ timeout_job_jid_sets[dep_name] << jid
230
+ end
231
+
232
+ def timeout_job_jid_sets
233
+ @timeout_job_jid_sets ||= Hash.new do |hash, dep|
234
+ key = [key_prefix, "timeout_job_jids", dep].join(':')
235
+ hash[dep] = Redis::Set.new(key, redis)
236
+ end
237
+ end
238
+
239
+ SpawnOptions = Struct.new(:data_overrides, :timeout_reduction)
240
+
241
+ def spawn_copy
242
+ options = SpawnOptions.new({})
243
+ yield options if block_given?
244
+ overrides = JSON.parse(JSON.dump options.data_overrides)
245
+
246
+ pipeline.enqueue_jobs_for(data.merge(overrides), {
247
+ spawned_from_id: id,
248
+ timeout_reduction: options.timeout_reduction || 0
249
+ })
250
+ end
251
+
252
+ private
253
+
254
+ def populate_meta_for_create(batch_data, options)
255
+ metadata = {
256
+ created_at: Time.now.getutc.iso8601,
257
+ timeout_reduction: 0,
258
+ BATCH_DATA_KEY => JSON.dump(batch_data),
259
+ creation_in_progress: 1
260
+ }.merge(options)
261
+
262
+ meta.fill(metadata)
263
+ @timeout_reduction = metadata.fetch(:timeout_reduction)
264
+ end
265
+
266
+ SomeJobsFailedToCancelError = Class.new(StandardError)
267
+ CreationInStillInProgressError = Class.new(StandardError)
268
+
269
+ def perform_cancellation
270
+ return true if cancelled?
271
+
272
+ if creation_in_progress?
273
+ raise CreationInStillInProgressError,
274
+ "#{id} is still being created (started " +
275
+ "#{Time.now - created_at} seconds ago)"
276
+ end
277
+
278
+ qless.bulk_cancel(job_jids)
279
+ verify_all_jobs_cancelled
280
+
281
+ external_deps.each do |key|
282
+ cancel_timeout_job_jid_set_for(key)
283
+ end
284
+
285
+ meta["cancelled"] = "1"
286
+ set_expiration!
287
+ pipeline.configuration.notify(:after_job_batch_cancellation, self)
288
+ end
289
+
290
+ def verify_all_jobs_cancelled
291
+ jobs = qless_jobs.reject { |j| j.state == "complete" }
292
+ return if jobs.none?
293
+
294
+ raise SomeJobsFailedToCancelError,
295
+ "#{jobs.size} jobs failed to cancel: #{jobs.inspect}"
296
+ end
297
+
298
+ def update_external_dependency(dep_name, meth, jids)
299
+ jids.each do |jid|
300
+ EnqueuedJob.new(qless, pipeline, jid).send(meth, dep_name)
301
+ end
302
+ end
303
+
304
+ def _complete?(pending_size, complete_size)
305
+ pending_size == 0 && complete_size > 0
306
+ end
307
+
308
+ def time_from(meta_entry)
309
+ date_string = meta[meta_entry]
310
+ Time.iso8601(date_string) if date_string
311
+ end
312
+
313
+ def set_expiration!
314
+ keys_to_expire = declared_redis_object_keys.to_set
315
+
316
+ each_enqueued_job do |job|
317
+ keys_to_expire.merge(job.declared_redis_object_keys)
318
+
319
+ job.all_external_dependencies.each do |dep|
320
+ keys_to_expire << external_dependency_sets[dep].key
321
+ keys_to_expire << timeout_job_jid_sets[dep].key
322
+ end
323
+ end
324
+
325
+ set_expiration_on(*keys_to_expire)
326
+ end
327
+
328
+ def each_enqueued_job
329
+ job_jids.each do |jid|
330
+ yield EnqueuedJob.new(qless, pipeline, jid)
331
+ end
332
+ end
333
+
334
+ def external_dependency_sets
335
+ @external_dependency_sets ||= Hash.new do |hash, dep|
336
+ key = [key_prefix, "ext_deps", dep].join(':')
337
+ hash[dep] = Redis::Set.new(key, redis)
338
+ end
339
+ end
340
+
341
+ def decode(string)
342
+ string && JSON.load(string)
343
+ end
344
+
345
+ def cancel_timeout_job_jid_set_for(dep_name)
346
+ timeout_job_jid_set = timeout_job_jid_sets[dep_name]
347
+ timeout_job_jid_set.each { |jid| gracefully_cancel(jid) }
348
+ timeout_job_jid_set.del
349
+ end
350
+
351
+ def gracefully_cancel(jid)
352
+ job = job_repository[jid]
353
+ job && job.cancel
354
+ end
355
+
356
+ def set_expiration_on(*redis_keys)
357
+ redis_keys.each do |key|
358
+ redis.pexpire(key, pipeline.configuration.data_ttl_in_milliseconds)
359
+ end
360
+ end
361
+ end
362
+ end
363
+