canvas_sync 0.17.3.beta3 → 0.17.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 54feabc8f9154b633a70a11d15bfeebaf54ab9d9eb24b9a636c27844728ca1a3
4
- data.tar.gz: 0c8d2692a83ebb37a518f61d11fb2970b3204f3d68c990c721fa23a7365a91b6
3
+ metadata.gz: 999dfad29e32b30e204cc26e6f36700752f3f3efc06314293c9d86df1b52b774
4
+ data.tar.gz: ef2c1b967f0451a3f2029d2aaae5dbe698e93a5a0136a52eeefe184dc1009f36
5
5
  SHA512:
6
- metadata.gz: 50efdad1e463d363463b72fa2fcc5581f877bdeb2c2440d688b10f451321996d95c7acd610d822fcc943d0220ed5520f4963de12341c1adb4238d131c5d8df04
7
- data.tar.gz: 51d40bc898a58a0aab65a85d4c1820ca5da72bf22b4c11e0ee631ad2e815dc344dc4824a2ade24e7485ace7ad0d12692d71cc93054afcfc05269ac22a4f7b4c9
6
+ metadata.gz: cd96c186dced2cb00a77fe8e729094ce8c9a7878f74c854c6e15fe1e143b773682b747065c231cf9babf777ebe752f9bdb24f130b7b2485dab9efae7e4ba1be9
7
+ data.tar.gz: c7d3f33ccbc78a5f4fc22ab1c834ebf21c2be0863e7ff6cbabb5af5ebb5f9181783d8a18f2a1a015d56193d0a67289e1f77f12dcd5942d0b42e3ef6e96b82317
@@ -99,7 +99,7 @@ module CanvasSync
99
99
  if parent_bid
100
100
  r.hincrby("BID-#{parent_bid}", "children", 1)
101
101
  r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
102
- r.sadd("BID-#{parent_bid}-bids", bid)
102
+ r.zadd("BID-#{parent_bid}-bids", created_at, bid)
103
103
  end
104
104
  end
105
105
  end
@@ -171,6 +171,13 @@ module CanvasSync
171
171
  @bidkey
172
172
  end
173
173
 
174
+ def flush_pending_attrs
175
+ super
176
+ redis do |r|
177
+ r.zadd("batches", created_at, bid)
178
+ end
179
+ end
180
+
174
181
  private
175
182
 
176
183
  def assert_batch_is_open
@@ -184,15 +191,16 @@ module CanvasSync
184
191
 
185
192
  def append_jobs(jids)
186
193
  jids = jids.uniq
194
+ return unless jids.size > 0
195
+
187
196
  redis do |r|
197
+ tme = Time.now.utc.to_f
198
+ added = r.zadd(@bidkey + "-jids", jids.map{|jid| [tme, jid] }, nx: true)
188
199
  r.multi do
189
- r.hincrby(@bidkey, "pending", jids.size)
200
+ r.hincrby(@bidkey, "pending", added)
201
+ r.hincrby(@bidkey, "job_count", added)
190
202
  r.expire(@bidkey, BID_EXPIRE_TTL)
191
-
192
- if jids.size > 0
193
- r.sadd(@bidkey + "-jids", jids)
194
- r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
195
- end
203
+ r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
196
204
  end
197
205
  end
198
206
  end
@@ -200,6 +208,8 @@ module CanvasSync
200
208
  class << self
201
209
  def process_failed_job(bid, jid)
202
210
  _, pending, failed, children, complete, parent_bid = redis do |r|
211
+ return unless r.exists?("BID-#{bid}")
212
+
203
213
  r.multi do
204
214
  r.sadd("BID-#{bid}-failed", jid)
205
215
 
@@ -220,6 +230,8 @@ module CanvasSync
220
230
 
221
231
  def process_dead_job(bid, jid)
222
232
  _, failed, children, complete, parent_bid = redis do |r|
233
+ return unless r.exists?("BID-#{bid}")
234
+
223
235
  r.multi do
224
236
  r.sadd("BID-#{bid}-dead", jid)
225
237
 
@@ -246,6 +258,8 @@ module CanvasSync
246
258
 
247
259
  def process_successful_job(bid, jid)
248
260
  _, failed, pending, children, complete, success, parent_bid = redis do |r|
261
+ return unless r.exists?("BID-#{bid}")
262
+
249
263
  r.multi do
250
264
  r.srem("BID-#{bid}-failed", jid)
251
265
 
@@ -257,13 +271,11 @@ module CanvasSync
257
271
  r.hget("BID-#{bid}", "parent_bid")
258
272
 
259
273
  r.hincrby("BID-#{bid}", "successful-jobs", 1)
260
- r.srem("BID-#{bid}-jids", jid)
274
+ r.zrem("BID-#{bid}-jids", jid)
261
275
  r.expire("BID-#{bid}", BID_EXPIRE_TTL)
262
276
  end
263
277
  end
264
278
 
265
- # TODO - There seems to be an issue where a :complete callback batch will occasionally run its job, but won't enqueue_callbacks()
266
-
267
279
  all_success = pending.to_i.zero? && children == success
268
280
  # if complete or successfull call complete callback (the complete callback may then call successful)
269
281
  if (pending.to_i == failed.to_i && children == complete) || all_success
@@ -302,6 +314,11 @@ module CanvasSync
302
314
  if callback_args.present? && !callback_params.present?
303
315
  logger.debug {"Enqueue callback bid: #{bid} event: #{event} args: #{callback_args.inspect}"}
304
316
 
317
+ redis do |r|
318
+ r.sadd("#{batch_key}-pending_callbacks", event)
319
+ r.expire("#{batch_key}-pending_callbacks", BID_EXPIRE_TTL)
320
+ end
321
+
305
322
  with_batch(parent_bid) do
306
323
  cb_batch = self.new
307
324
  cb_batch.callback_params = {
@@ -330,7 +347,8 @@ module CanvasSync
330
347
  def cleanup_redis(bid)
331
348
  logger.debug {"Cleaning redis of batch #{bid}"}
332
349
  redis do |r|
333
- r.del(
350
+ r.zrem("batches", bid)
351
+ r.unlink(
334
352
  "BID-#{bid}",
335
353
  "BID-#{bid}-callbacks-complete",
336
354
  "BID-#{bid}-callbacks-success",
@@ -341,10 +359,21 @@ module CanvasSync
341
359
  "BID-#{bid}-batches-failed",
342
360
  "BID-#{bid}-bids",
343
361
  "BID-#{bid}-jids",
362
+ "BID-#{bid}-pending_callbacks",
344
363
  )
345
364
  end
346
365
  end
347
366
 
367
+ def delete_prematurely!(bid)
368
+ child_bids = redis do |r|
369
+ r.zrange("BID-#{bid}-bids", 0, -1)
370
+ end
371
+ child_bids.each do |cbid|
372
+ delete_prematurely!(cbid)
373
+ end
374
+ cleanup_redis(bid)
375
+ end
376
+
348
377
  def redis(*args, &blk)
349
378
  defined?(::Sidekiq) ? ::Sidekiq.redis(*args, &blk) : nil # TODO
350
379
  end
@@ -353,8 +382,6 @@ module CanvasSync
353
382
  defined?(::Sidekiq) ? ::Sidekiq.logger : Rails.logger
354
383
  end
355
384
 
356
- private
357
-
358
385
  def push_callbacks(args, queue)
359
386
  Batch::Callback::worker_class.enqueue_all(args, queue)
360
387
  end
@@ -52,9 +52,6 @@ module CanvasSync
52
52
 
53
53
  class Finalize
54
54
  def dispatch(status, opts)
55
- is_callback_batch = opts['origin'].present?
56
- has_callback_batch = opts['callback_bid'].present?
57
-
58
55
  bid = opts["bid"]
59
56
  event = opts["event"].to_sym
60
57
 
@@ -63,12 +60,20 @@ module CanvasSync
63
60
  batch_status = Status.new bid
64
61
  send(event, bid, batch_status, batch_status.parent_bid)
65
62
 
66
- if event == :success && !has_callback_batch
67
- Batch.cleanup_redis(bid)
68
- end
69
-
70
- if event == :success && is_callback_batch && opts['origin']['event'].to_sym == :success
71
- Batch.cleanup_redis(opts['origin']['for_bid'])
63
+ if event == :success
64
+ if opts['origin'].present?
65
+ origin_bid = opts['origin']['for_bid']
66
+ _, pending, success_ran = Batch.redis do |r|
67
+ r.multi do
68
+ r.srem("BID-#{origin_bid}-pending_callbacks", opts['origin']['event'])
69
+ r.scard("BID-#{origin_bid}-pending_callbacks")
70
+ r.hget("BID-#{origin_bid}", "success")
71
+ end
72
+ end
73
+ Batch.cleanup_redis(origin_bid) if pending == 0 && success_ran == 'true'
74
+ elsif (Batch.redis {|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
75
+ Batch.cleanup_redis(bid)
76
+ end
72
77
  end
73
78
  end
74
79
 
@@ -113,13 +118,7 @@ module CanvasSync
113
118
  end
114
119
  end
115
120
 
116
- # If the batch was successful run :success callback, which will call the parent's :complete callback (if necessary)
117
- # Also, only trigger the success callback if the :complete callback_batch was successful
118
- if pending.to_i.zero? && children == success
119
- # Batch.enqueue_callbacks(:success, bid)
120
-
121
- # otherwise check for a parent and call its :complete if needed
122
- elsif parent_bid
121
+ if parent_bid && !(pending.to_i.zero? && children == success)
123
122
  # if batch was not successfull check and see if its parent is complete
124
123
  # if the parent is complete we trigger the complete callback
125
124
  # We don't want to run this if the batch was successfull because the success
@@ -28,6 +28,10 @@ module CanvasSync
28
28
  @hash_map[local_bid]
29
29
  end
30
30
 
31
+ def own
32
+ resolve_hash(@bid_stack[-1]) || {}
33
+ end
34
+
31
35
  def set_local(new_hash)
32
36
  @dirty = true
33
37
  local.clear.merge!(new_hash)
@@ -106,7 +110,8 @@ module CanvasSync
106
110
  @bid_stack[index]
107
111
  else
108
112
  pbid = Batch.redis do |r|
109
- r.hget("BID-#{bid}", "parent_bid") || r.hget("BID-#{bid}", "callback_for")
113
+ callback_params = JSON.parse(r.hget("BID-#{bid}", "callback_params") || "{}")
114
+ callback_params['for_bid'] || r.hget("BID-#{bid}", "parent_bid")
110
115
  end
111
116
  @bid_stack.unshift(pbid)
112
117
  pbid
@@ -24,6 +24,7 @@ module CanvasSync
24
24
  end
25
25
 
26
26
  def self.from_pid(pid)
27
+ raise "PID must be given" unless pid.present?
27
28
  new(pid)
28
29
  end
29
30
 
@@ -38,7 +39,7 @@ module CanvasSync
38
39
  def add_jobs(job_descs)
39
40
  job_descs.each do |job_desc|
40
41
  wrapper = Batch.new
41
- wrapper.description = "Pool Job Wrapper"
42
+ wrapper.description = "Pool Job Wrapper (PID: #{pid})"
42
43
  checkin_event = (on_failed_job == :wait) ? :success : :complete
43
44
  wrapper.on(checkin_event, "#{self.class.to_s}.job_checked_in", pool_id: pid)
44
45
  wrapper.jobs {}
@@ -57,19 +58,42 @@ module CanvasSync
57
58
  def cleanup_redis
58
59
  Batch.logger.debug {"Cleaning redis of pool #{pid}"}
59
60
  redis do |r|
60
- r.del(
61
+ r.zrem("pools", pid)
62
+ r.unlink(
61
63
  "#{redis_key}",
62
64
  "#{redis_key}-jobs",
63
65
  )
64
66
  end
65
67
  end
66
68
 
69
+ def active_count
70
+ redis do |r|
71
+ r.hincrby(redis_key, "active_count", 0)
72
+ end
73
+ end
74
+
75
+ def pending_count
76
+ jobs_key = "#{redis_key}-jobs"
77
+ order = self.order || 'fifo'
78
+ redis do |r|
79
+ case order.to_sym
80
+ when :fifo, :lifo
81
+ r.llen(jobs_key)
82
+ when :random
83
+ r.scard(jobs_key)
84
+ when :priority
85
+ r.zcard(jobs_key)
86
+ end
87
+ end
88
+ end
89
+
67
90
  def job_checked_in(status, options)
68
91
  active_count = redis do |r|
92
+ return unless r.exists?(redis_key)
69
93
  r.hincrby(redis_key, "active_count", -1)
70
94
  end
71
- added_count = refill_allotment
72
95
 
96
+ added_count = refill_allotment
73
97
  if active_count == 0 && added_count == 0
74
98
  cleanup_redis if clean_when_empty
75
99
  end
@@ -159,29 +183,25 @@ module CanvasSync
159
183
  ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]
160
184
  end
161
185
 
162
- def pending_count
163
- order = self.order
164
- redis do |r|
165
- case order.to_sym
166
- when :fifo, :lifo
167
- r.llen(jobs_key)
168
- when :random
169
- r.scard(jobs_key)
170
- when :priority
171
- r.zcard(jobs_key)
172
- end
173
- end
174
- end
175
-
176
186
  def self.redis(&blk)
177
187
  Batch.redis &blk
178
188
  end
179
189
  delegate :redis, to: :class
180
190
 
191
+ protected
192
+
193
+ def flush_pending_attrs
194
+ super
195
+ redis do |r|
196
+ r.zadd("pools", created_at, pid)
197
+ end
198
+ end
199
+
181
200
  private
182
201
 
183
- def initialize_new(concurrency: nil, order: :fifo, clean_when_empty: true, on_failed_job: :wait)
202
+ def initialize_new(concurrency: nil, order: :fifo, clean_when_empty: true, on_failed_job: :wait, description: nil)
184
203
  self.created_at = Time.now.utc.to_f
204
+ self.description = description
185
205
  self.order = order
186
206
  self.concurrency = concurrency
187
207
  self.clean_when_empty = clean_when_empty
@@ -117,3 +117,5 @@ module CanvasSync
117
117
  end
118
118
  end
119
119
  end
120
+
121
+ require_relative 'sidekiq/web'
@@ -0,0 +1,114 @@
1
+
2
+ begin
3
+ require "sidekiq/web"
4
+ rescue LoadError
5
+ # client-only usage
6
+ end
7
+
8
+ require_relative "web/helpers"
9
+
10
+ module CanvasSync::JobBatches::Sidekiq
11
+ module Web
12
+ def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
13
+ app.helpers do
14
+ include Web::Helpers
15
+ end
16
+
17
+ app.get "/batches" do
18
+ @count = (params['count'] || 25).to_i
19
+ @current_page, @total_size, @batches = page('batches', params['page'], @count)
20
+ @batches = @batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
21
+
22
+ erb(get_template(:batches))
23
+ end
24
+
25
+ app.get "/batches/:bid" do
26
+ @bid = params[:bid]
27
+ @batch = CanvasSync::JobBatches::Batch.new(@bid)
28
+
29
+ @count = (params['count'] || 25).to_i
30
+ @current_batches_page, @total_batches_size, @sub_batches = page("BID-#{@batch.bid}-bids", params['batch_page'], @count)
31
+ @sub_batches = @sub_batches.map {|b, score| CanvasSync::JobBatches::Batch.new(b) }
32
+
33
+ @current_jobs_page, @total_jobs_size, @jobs = page("BID-#{@batch.bid}-jids", params['job_page'], @count)
34
+ @jobs = @jobs.map {|jid, score| jid }
35
+
36
+ erb(get_template(:batch))
37
+ end
38
+
39
+ app.post "/batches/all" do
40
+ if params['delete']
41
+ drain_zset('batches') do |batches|
42
+ batches.each do |bid|
43
+ CanvasSync::JobBatches::Batch.cleanup_redis(bid)
44
+ end
45
+ end
46
+ end
47
+
48
+ redirect "#{root_path}batches"
49
+ end
50
+
51
+ app.post "/batches/:bid" do
52
+ @bid = params[:bid]
53
+ @batch = CanvasSync::JobBatches::Batch.new(@bid)
54
+
55
+ if params['delete']
56
+ CanvasSync::JobBatches::Batch.delete_prematurely!(@bid)
57
+ end
58
+
59
+ redirect_with_query("#{root_path}batches")
60
+ end
61
+
62
+ # =============== POOLS =============== #
63
+
64
+ app.get "/pools" do
65
+ @count = (params['count'] || 25).to_i
66
+ @current_page, @total_size, @pools = page('pools', params['page'], @count)
67
+ @pools = @pools.map {|b, score| CanvasSync::JobBatches::Pool.new(b) }
68
+
69
+ erb(get_template(:pools))
70
+ end
71
+
72
+ app.get "/pools/:pid" do
73
+ @pid = params[:pid]
74
+ @pool = CanvasSync::JobBatches::Pool.new(@pid)
75
+
76
+ @count = (params['count'] || 25).to_i
77
+ @current_jobs_page, @total_jobs_size, @jobs = page("POOLID-#{@pool.pid}-jobs", params['job_page'], @count)
78
+ @jobs = @jobs.map {|desc, score=nil| JSON.parse(desc)[0] }
79
+
80
+ erb(get_template(:pool))
81
+ end
82
+
83
+ app.post "/pools/all" do
84
+ if params['delete']
85
+ drain_zset('pools') do |pools|
86
+ pools.each do |pid|
87
+ CanvasSync::JobBatches::Pool.from_pid(pid).cleanup_redis
88
+ end
89
+ end
90
+ end
91
+
92
+ redirect "#{root_path}pools"
93
+ end
94
+
95
+ app.post "/pools/:pid" do
96
+ @pid = params[:pid]
97
+ @pool = CanvasSync::JobBatches::Pool.from_pid(@pid)
98
+
99
+ if params['delete']
100
+ @pool.cleanup_redis
101
+ end
102
+
103
+ redirect_with_query("#{root_path}pools")
104
+ end
105
+ end
106
+ end
107
+ end
108
+
109
+ if defined?(::Sidekiq::Web)
110
+ ::Sidekiq::Web.register CanvasSync::JobBatches::Sidekiq::Web
111
+ ::Sidekiq::Web.tabs["Batches"] = "batches"
112
+ ::Sidekiq::Web.tabs["Pools"] = "pools"
113
+ ::Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
114
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ module CanvasSync::JobBatches::Sidekiq
4
+ module Web
5
+ module Helpers
6
+ VIEW_PATH = File.expand_path("../web/views", __dir__)
7
+
8
+ module_function
9
+
10
+ def get_template(name)
11
+ File.open(File.join(VIEW_PATH, "#{name}.erb")).read
12
+ end
13
+
14
+ def drain_zset(key)
15
+ items, _ = Sidekiq.redis do |r|
16
+ r.multi do
17
+ r.zrange(key, 0, -1)
18
+ r.zremrangebyrank(key, 0, -1)
19
+ end
20
+ end
21
+ yield items
22
+ end
23
+
24
+ def safe_relative_time(time)
25
+ time = parse_time(time)
26
+ relative_time(time)
27
+ end
28
+
29
+ def parse_time(time)
30
+ case time
31
+ when Time
32
+ time
33
+ when Integer, Float
34
+ Time.at(time)
35
+ else
36
+ Time.parse(time.to_s)
37
+ end
38
+ end
39
+ end
40
+ end
41
+ end