canvas_sync 0.17.1 → 0.17.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +58 -0
  3. data/lib/canvas_sync/job_batches/batch.rb +101 -115
  4. data/lib/canvas_sync/job_batches/callback.rb +29 -34
  5. data/lib/canvas_sync/job_batches/context_hash.rb +13 -5
  6. data/lib/canvas_sync/job_batches/hincr_max.lua +5 -0
  7. data/lib/canvas_sync/job_batches/jobs/managed_batch_job.rb +99 -0
  8. data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb +6 -65
  9. data/lib/canvas_sync/job_batches/pool.rb +213 -0
  10. data/lib/canvas_sync/job_batches/redis_model.rb +69 -0
  11. data/lib/canvas_sync/job_batches/redis_script.rb +163 -0
  12. data/lib/canvas_sync/job_batches/sidekiq.rb +24 -1
  13. data/lib/canvas_sync/job_batches/sidekiq/web.rb +114 -0
  14. data/lib/canvas_sync/job_batches/sidekiq/web/helpers.rb +41 -0
  15. data/lib/canvas_sync/job_batches/sidekiq/web/views/_batches_table.erb +42 -0
  16. data/lib/canvas_sync/job_batches/sidekiq/web/views/_pagination.erb +26 -0
  17. data/lib/canvas_sync/job_batches/sidekiq/web/views/batch.erb +138 -0
  18. data/lib/canvas_sync/job_batches/sidekiq/web/views/batches.erb +23 -0
  19. data/lib/canvas_sync/job_batches/sidekiq/web/views/pool.erb +85 -0
  20. data/lib/canvas_sync/job_batches/sidekiq/web/views/pools.erb +47 -0
  21. data/lib/canvas_sync/job_batches/status.rb +9 -5
  22. data/lib/canvas_sync/jobs/begin_sync_chain_job.rb +3 -1
  23. data/lib/canvas_sync/version.rb +1 -1
  24. data/spec/dummy/log/test.log +140455 -0
  25. data/spec/job_batching/batch_aware_job_spec.rb +1 -0
  26. data/spec/job_batching/batch_spec.rb +72 -16
  27. data/spec/job_batching/callback_spec.rb +1 -1
  28. data/spec/job_batching/context_hash_spec.rb +54 -0
  29. data/spec/job_batching/flow_spec.rb +5 -11
  30. data/spec/job_batching/integration/fail_then_succeed.rb +42 -0
  31. data/spec/job_batching/integration_helper.rb +6 -4
  32. data/spec/job_batching/sidekiq_spec.rb +1 -0
  33. data/spec/job_batching/status_spec.rb +4 -20
  34. data/spec/spec_helper.rb +3 -7
  35. metadata +19 -16
@@ -2,6 +2,7 @@ module CanvasSync
2
2
  module JobBatches
3
3
  class Batch
4
4
  module Callback
5
+ mattr_accessor :worker_class
5
6
 
6
7
  VALID_CALLBACKS = %w[success complete dead].freeze
7
8
 
@@ -47,47 +48,39 @@ module CanvasSync
47
48
  end
48
49
  end
49
50
 
50
- if defined?(::Sidekiq)
51
- class SidekiqCallbackWorker
52
- include ::Sidekiq::Worker
53
- include CallbackWorkerCommon
54
-
55
- def self.enqueue_all(args, queue)
56
- return if args.empty?
57
-
58
- ::Sidekiq::Client.push_bulk(
59
- 'class' => self,
60
- 'args' => args,
61
- 'queue' => queue
62
- )
63
- end
64
- end
65
- Worker = SidekiqCallbackWorker
66
- else
67
- Worker = ActiveJobCallbackWorker
68
- end
51
+ worker_class = ActiveJobCallbackWorker
69
52
 
70
53
  class Finalize
71
- def dispatch status, opts
54
+ def dispatch(status, opts)
72
55
  bid = opts["bid"]
73
- callback_bid = status.bid
74
56
  event = opts["event"].to_sym
75
- callback_batch = bid != callback_bid
76
57
 
77
- Batch.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}, callback batch id: #{callback_bid} callback_batch #{callback_batch}"}
58
+ Batch.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}, callback batch id: #{callback_bid} callback_batch #{is_callback_batch}"}
78
59
 
79
60
  batch_status = Status.new bid
80
61
  send(event, bid, batch_status, batch_status.parent_bid)
81
62
 
82
- # Different events are run in different callback batches
83
- Batch.cleanup_redis callback_bid if callback_batch
84
- Batch.cleanup_redis bid if event == :success
63
+ if event == :success
64
+ if opts['origin'].present?
65
+ origin_bid = opts['origin']['for_bid']
66
+ _, pending, success_ran = Batch.redis do |r|
67
+ r.multi do
68
+ r.srem("BID-#{origin_bid}-pending_callbacks", opts['origin']['event'])
69
+ r.scard("BID-#{origin_bid}-pending_callbacks")
70
+ r.hget("BID-#{origin_bid}", "success")
71
+ end
72
+ end
73
+ Batch.cleanup_redis(origin_bid) if pending == 0 && success_ran == 'true'
74
+ elsif (Batch.redis {|r| r.scard("BID-#{bid}-pending_callbacks") }) == 0
75
+ Batch.cleanup_redis(bid)
76
+ end
77
+ end
85
78
  end
86
79
 
87
80
  def success(bid, status, parent_bid)
88
81
  return unless parent_bid
89
82
 
90
- _, _, success, _, _, complete, pending, children, failure = Batch.redis do |r|
83
+ _, _, success, _, _, complete, pending, children, success, failure = Batch.redis do |r|
91
84
  r.multi do
92
85
  r.sadd("BID-#{parent_bid}-batches-success", bid)
93
86
  r.expire("BID-#{parent_bid}-batches-success", Batch::BID_EXPIRE_TTL)
@@ -99,15 +92,21 @@ module CanvasSync
99
92
 
100
93
  r.hincrby("BID-#{parent_bid}", "pending", 0)
101
94
  r.hincrby("BID-#{parent_bid}", "children", 0)
95
+ r.scard("BID-#{parent_bid}-batches-success")
102
96
  r.scard("BID-#{parent_bid}-failed")
103
97
  end
104
98
  end
105
- # if job finished successfully and parent batch completed call parent complete callback
106
- # Success callback is called after complete callback
99
+
100
+ # If the job finished successfully and parent batch is completed, call parent :complete callback
101
+ # Parent :success callback will be called by its :complete callback
107
102
  if complete == children && pending == failure
108
103
  Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
109
104
  Batch.enqueue_callbacks(:complete, parent_bid)
110
105
  end
106
+ if pending.to_i.zero? && children == success
107
+ Batch.logger.debug {"Finalize parent success bid: #{parent_bid}"}
108
+ Batch.enqueue_callbacks(:success, parent_bid)
109
+ end
111
110
  end
112
111
 
113
112
  def complete(bid, status, parent_bid)
@@ -119,11 +118,7 @@ module CanvasSync
119
118
  end
120
119
  end
121
120
 
122
- # if we batch was successful run success callback
123
- if pending.to_i.zero? && children == success
124
- Batch.enqueue_callbacks(:success, bid)
125
-
126
- elsif parent_bid
121
+ if parent_bid && !(pending.to_i.zero? && children == success)
127
122
  # if batch was not successfull check and see if its parent is complete
128
123
  # if the parent is complete we trigger the complete callback
129
124
  # We don't want to run this if the batch was successfull because the success
@@ -17,7 +17,7 @@ module CanvasSync
17
17
  def local_bid
18
18
  bid = @bid_stack[-1]
19
19
  while bid.present?
20
- bhash = reolve_hash(bid)
20
+ bhash = resolve_hash(bid)
21
21
  return bid if bhash
22
22
  bid = get_parent_bid(bid)
23
23
  end
@@ -28,6 +28,10 @@ module CanvasSync
28
28
  @hash_map[local_bid]
29
29
  end
30
30
 
31
+ def own
32
+ resolve_hash(@bid_stack[-1]) || {}
33
+ end
34
+
31
35
  def set_local(new_hash)
32
36
  @dirty = true
33
37
  local.clear.merge!(new_hash)
@@ -49,7 +53,7 @@ module CanvasSync
49
53
  def [](key)
50
54
  bid = @bid_stack[-1]
51
55
  while bid.present?
52
- bhash = reolve_hash(bid)
56
+ bhash = resolve_hash(bid)
53
57
  return bhash[key] if bhash&.key?(key)
54
58
  bid = get_parent_bid(bid)
55
59
  end
@@ -94,7 +98,7 @@ module CanvasSync
94
98
  private
95
99
 
96
100
  def get_parent_hash(bid)
97
- reolve_hash(get_parent_bid(bid)).freeze
101
+ resolve_hash(get_parent_bid(bid)).freeze
98
102
  end
99
103
 
100
104
  def get_parent_bid(bid)
@@ -105,13 +109,16 @@ module CanvasSync
105
109
  if index >= 0
106
110
  @bid_stack[index]
107
111
  else
108
- pbid = Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
112
+ pbid = Batch.redis do |r|
113
+ callback_params = JSON.parse(r.hget("BID-#{bid}", "callback_params") || "{}")
114
+ callback_params['for_bid'] || r.hget("BID-#{bid}", "parent_bid")
115
+ end
109
116
  @bid_stack.unshift(pbid)
110
117
  pbid
111
118
  end
112
119
  end
113
120
 
114
- def reolve_hash(bid)
121
+ def resolve_hash(bid)
115
122
  return nil unless bid.present?
116
123
  return @hash_map[bid] if @hash_map.key?(bid)
117
124
 
@@ -137,6 +144,7 @@ module CanvasSync
137
144
  end
138
145
 
139
146
  def load_all
147
+ resolve_hash(@bid_stack[0]).freeze
140
148
  while @bid_stack[0].present?
141
149
  get_parent_hash(@bid_stack[0])
142
150
  end
@@ -0,0 +1,5 @@
1
+ local r=redis.call('HGET', KEYS[1], ARGV[1])
2
+ if r == false or r < ARGV[2] then
3
+ redis.call('HINCRBY', KEYS[1], ARGV[1], 1)
4
+ end
5
+ return r or 0
@@ -0,0 +1,99 @@
1
+ require_relative './base_job'
2
+
3
+ module CanvasSync
4
+ module JobBatches
5
+ class ManagedBatchJob < BaseJob
6
+ def perform(sub_jobs, context: nil, ordered: true, concurrency: nil)
7
+ man_batch_id = SecureRandom.urlsafe_base64(10)
8
+
9
+ if concurrency == 0 || concurrency == nil || concurrency == true
10
+ concurrency = sub_jobs.count
11
+ elsif concurrency == false
12
+ concurrency = 1
13
+ end
14
+
15
+ root_batch = Batch.new
16
+
17
+ Batch.redis do |r|
18
+ r.multi do
19
+ r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
20
+ r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
21
+ r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
22
+ r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
23
+
24
+ mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
25
+ j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
26
+ j = ActiveJob::Arguments.serialize([j])
27
+ JSON.unparse(j)
28
+ end
29
+ if ordered
30
+ r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
31
+ else
32
+ r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
33
+ end
34
+ r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
35
+ end
36
+ end
37
+
38
+ root_batch.description = "Managed Batch Root (#{man_batch_id})"
39
+ root_batch.allow_context_changes = (concurrency == 1)
40
+ root_batch.context = context
41
+ root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", managed_batch_id: man_batch_id)
42
+ root_batch.jobs {}
43
+
44
+ concurrency.times do
45
+ self.class.perform_next_sequence_job(man_batch_id)
46
+ end
47
+ end
48
+
49
+ def self.cleanup_redis(status, options)
50
+ man_batch_id = options['managed_batch_id']
51
+ Batch.redis do |r|
52
+ r.del(
53
+ "MNGBID-#{man_batch_id}",
54
+ "MNGBID-#{man_batch_id}-jobs",
55
+ )
56
+ end
57
+ end
58
+
59
+ def self.job_succeeded_callback(status, options)
60
+ man_batch_id = options['managed_batch_id']
61
+ perform_next_sequence_job(man_batch_id)
62
+ end
63
+
64
+ protected
65
+
66
+ def self.perform_next_sequence_job(man_batch_id)
67
+ root_bid, ordered = Batch.redis do |r|
68
+ r.multi do
69
+ r.hget("MNGBID-#{man_batch_id}", "root_bid")
70
+ r.hget("MNGBID-#{man_batch_id}", "ordered")
71
+ end
72
+ end
73
+
74
+ next_job_json = Batch.redis do |r|
75
+ if ordered
76
+ r.lpop("MNGBID-#{man_batch_id}-jobs")
77
+ else
78
+ r.spop("MNGBID-#{man_batch_id}-jobs")
79
+ end
80
+ end
81
+
82
+ return unless next_job_json.present?
83
+
84
+ next_job = JSON.parse(next_job_json)
85
+ next_job = ActiveJob::Arguments.deserialize(next_job)[0]
86
+
87
+ Batch.new(root_bid).jobs do
88
+ Batch.new.tap do |batch|
89
+ batch.description = "Managed Batch Fiber (#{man_batch_id})"
90
+ batch.on(:success, "#{self.to_s}.job_succeeded_callback", managed_batch_id: man_batch_id)
91
+ batch.jobs do
92
+ ChainBuilder.enqueue_job(next_job)
93
+ end
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
99
+ end
@@ -4,71 +4,12 @@ module CanvasSync
4
4
  module JobBatches
5
5
  class SerialBatchJob < BaseJob
6
6
  def perform(sub_jobs, context: nil)
7
- serial_id = SecureRandom.urlsafe_base64(10)
8
-
9
- root_batch = Batch.new
10
-
11
- Batch.redis do |r|
12
- r.multi do
13
- mapped_sub_jobs = sub_jobs.map do |j|
14
- j = ActiveJob::Arguments.serialize([j])
15
- JSON.unparse(j)
16
- end
17
- r.hset("SERBID-#{serial_id}", "root_bid", root_batch.bid)
18
- r.expire("SERBID-#{serial_id}", Batch::BID_EXPIRE_TTL)
19
- r.rpush("SERBID-#{serial_id}-jobs", mapped_sub_jobs)
20
- r.expire("SERBID-#{serial_id}-jobs", Batch::BID_EXPIRE_TTL)
21
- end
22
- end
23
-
24
- root_batch.description = "Serial Batch Root (#{serial_id})"
25
- root_batch.allow_context_changes = true
26
- root_batch.context = context
27
- root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", serial_batch_id: serial_id)
28
- root_batch.jobs {}
29
-
30
- self.class.perform_next_sequence_job(serial_id)
31
- end
32
-
33
- def self.cleanup_redis(status, options)
34
- serial_id = options['serial_batch_id']
35
- Batch.redis do |r|
36
- r.del(
37
- "SERBID-#{serial_id}",
38
- "SERBID-#{serial_id}-jobs",
39
- )
40
- end
41
- end
42
-
43
- def self.job_succeeded_callback(status, options)
44
- serial_id = options['serial_batch_id']
45
- perform_next_sequence_job(serial_id)
46
- end
47
-
48
- protected
49
-
50
- def self.perform_next_sequence_job(serial_id)
51
- root_bid, next_job_json = Batch.redis do |r|
52
- r.multi do
53
- r.hget("SERBID-#{serial_id}", "root_bid")
54
- r.lpop("SERBID-#{serial_id}-jobs")
55
- end
56
- end
57
-
58
- return unless next_job_json.present?
59
-
60
- next_job = JSON.parse(next_job_json)
61
- next_job = ActiveJob::Arguments.deserialize(next_job)[0]
62
-
63
- Batch.new(root_bid).jobs do
64
- Batch.new.tap do |batch|
65
- batch.description = "Serial Batch Fiber (#{serial_id})"
66
- batch.on(:success, "#{self.to_s}.job_succeeded_callback", serial_batch_id: serial_id)
67
- batch.jobs do
68
- ChainBuilder.enqueue_job(next_job)
69
- end
70
- end
71
- end
7
+ ManagedBatchJob.new.perform(
8
+ sub_jobs,
9
+ context: context,
10
+ ordered: true,
11
+ concurrency: false,
12
+ )
72
13
  end
73
14
  end
74
15
  end
@@ -0,0 +1,213 @@
1
+ module CanvasSync
2
+ module JobBatches
3
+ class Pool
4
+ include RedisModel
5
+
6
+ HINCR_MAX = RedisScript.new(Pathname.new(__FILE__) + "../hincr_max.lua")
7
+
8
+ attr_reader :pid
9
+ redis_attr :description
10
+ redis_attr :created_at
11
+ redis_attr :concurrency, :int
12
+ redis_attr :order
13
+ redis_attr :on_failed_job, :symbol
14
+ redis_attr :clean_when_empty, :bool
15
+
16
+ def initialize(pooolid = nil, **kwargs)
17
+ if pooolid
18
+ @existing = true
19
+ @pid = pooolid
20
+ else
21
+ @pid = SecureRandom.urlsafe_base64(10)
22
+ initialize_new(**kwargs)
23
+ end
24
+ end
25
+
26
+ def self.from_pid(pid)
27
+ raise "PID must be given" unless pid.present?
28
+ new(pid)
29
+ end
30
+
31
+ def <<(job_desc)
32
+ add_job(job_desc)
33
+ end
34
+
35
+ def add_job(job_desc)
36
+ add_jobs([job_desc])
37
+ end
38
+
39
+ def add_jobs(job_descs)
40
+ job_descs.each do |job_desc|
41
+ wrapper = Batch.new
42
+ wrapper.description = "Pool Job Wrapper (PID: #{pid})"
43
+ checkin_event = (on_failed_job == :wait) ? :success : :complete
44
+ wrapper.on(checkin_event, "#{self.class.to_s}.job_checked_in", pool_id: pid)
45
+ wrapper.jobs {}
46
+
47
+ job_desc = job_desc.with_indifferent_access
48
+ job_desc = job_desc.merge!(
49
+ job: job_desc[:job].to_s,
50
+ pool_wrapper_batch: wrapper.bid,
51
+ )
52
+
53
+ push_job_to_pool(job_desc)
54
+ end
55
+ refill_allotment
56
+ end
57
+
58
+ def cleanup_redis
59
+ Batch.logger.debug {"Cleaning redis of pool #{pid}"}
60
+ redis do |r|
61
+ r.zrem("pools", pid)
62
+ r.unlink(
63
+ "#{redis_key}",
64
+ "#{redis_key}-jobs",
65
+ )
66
+ end
67
+ end
68
+
69
+ def active_count
70
+ redis do |r|
71
+ r.hincrby(redis_key, "active_count", 0)
72
+ end
73
+ end
74
+
75
+ def pending_count
76
+ jobs_key = "#{redis_key}-jobs"
77
+ order = self.order || 'fifo'
78
+ redis do |r|
79
+ case order.to_sym
80
+ when :fifo, :lifo
81
+ r.llen(jobs_key)
82
+ when :random
83
+ r.scard(jobs_key)
84
+ when :priority
85
+ r.zcard(jobs_key)
86
+ end
87
+ end
88
+ end
89
+
90
+ def job_checked_in(status, options)
91
+ active_count = redis do |r|
92
+ return unless r.exists?(redis_key)
93
+ r.hincrby(redis_key, "active_count", -1)
94
+ end
95
+
96
+ added_count = refill_allotment
97
+ if active_count == 0 && added_count == 0
98
+ cleanup_redis if clean_when_empty
99
+ end
100
+ end
101
+
102
+ def self.job_checked_in(status, options)
103
+ pid = options['pool_id']
104
+ from_pid(pid).job_checked_in(status, options)
105
+ end
106
+
107
+ protected
108
+
109
+ def redis_key
110
+ "POOLID-#{pid}"
111
+ end
112
+
113
+ def refill_allotment
114
+ jobs_added = 0
115
+ limit = concurrency.to_i
116
+ redis do |r|
117
+ current_count = 0
118
+ while true
119
+ current_count = HINCR_MAX.call(r, [redis_key], ["active_count", limit]).to_i
120
+ if current_count < limit
121
+ job_desc = pop_job_from_pool
122
+ if job_desc.present?
123
+ Batch.new(job_desc['pool_wrapper_batch']).jobs do
124
+ ChainBuilder.enqueue_job(job_desc)
125
+ end
126
+ jobs_added += 1
127
+ else
128
+ r.hincrby(redis_key, "active_count", -1)
129
+ break
130
+ end
131
+ else
132
+ break
133
+ end
134
+ end
135
+ r.expire(redis_key, Batch::BID_EXPIRE_TTL)
136
+ r.expire("#{redis_key}-jobs", Batch::BID_EXPIRE_TTL)
137
+ end
138
+ jobs_added
139
+ end
140
+
141
+ def push_job_to_pool(job_desc)
142
+ jobs_key = "#{redis_key}-jobs"
143
+ # This allows duplicate jobs when a Redis Set is used
144
+ job_desc['_pool_random_key_'] = SecureRandom.urlsafe_base64(10)
145
+ job_json = JSON.unparse(ActiveJob::Arguments.serialize([job_desc]))
146
+ order = self.order
147
+
148
+ redis do |r|
149
+ r.multi do
150
+ case order.to_sym
151
+ when :fifo, :lifo
152
+ r.rpush(jobs_key, job_json)
153
+ when :random
154
+ r.sadd(jobs_key, job_json)
155
+ when :priority
156
+ r.zadd(jobs_key, job_desc[:priority] || 0, job_json)
157
+ end
158
+ r.expire(jobs_key, Batch::BID_EXPIRE_TTL)
159
+ end
160
+ end
161
+ end
162
+
163
+ def pop_job_from_pool
164
+ jobs_key = "#{redis_key}-jobs"
165
+ order = self.order
166
+
167
+ job_json = nil
168
+ redis do |r|
169
+ job_json = case order.to_sym
170
+ when :fifo
171
+ r.lpop(jobs_key)
172
+ when :lifo
173
+ r.rpop(jobs_key)
174
+ when :random
175
+ r.spop(jobs_key)
176
+ when :priority
177
+ r.zpopmax(jobs_key)
178
+ end
179
+ end
180
+
181
+ return nil unless job_json.present?
182
+
183
+ ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]
184
+ end
185
+
186
+ def self.redis(&blk)
187
+ Batch.redis &blk
188
+ end
189
+ delegate :redis, to: :class
190
+
191
+ protected
192
+
193
+ def flush_pending_attrs
194
+ super
195
+ redis do |r|
196
+ r.zadd("pools", created_at, pid)
197
+ end
198
+ end
199
+
200
+ private
201
+
202
+ def initialize_new(concurrency: nil, order: :fifo, clean_when_empty: true, on_failed_job: :wait, description: nil)
203
+ self.created_at = Time.now.utc.to_f
204
+ self.description = description
205
+ self.order = order
206
+ self.concurrency = concurrency
207
+ self.clean_when_empty = clean_when_empty
208
+ self.on_failed_job = on_failed_job
209
+ flush_pending_attrs
210
+ end
211
+ end
212
+ end
213
+ end