canvas_sync 0.17.0 → 0.17.3.beta3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -17,7 +17,7 @@ module CanvasSync
17
17
  def local_bid
18
18
  bid = @bid_stack[-1]
19
19
  while bid.present?
20
- bhash = reolve_hash(bid)
20
+ bhash = resolve_hash(bid)
21
21
  return bid if bhash
22
22
  bid = get_parent_bid(bid)
23
23
  end
@@ -49,7 +49,7 @@ module CanvasSync
49
49
  def [](key)
50
50
  bid = @bid_stack[-1]
51
51
  while bid.present?
52
- bhash = reolve_hash(bid)
52
+ bhash = resolve_hash(bid)
53
53
  return bhash[key] if bhash&.key?(key)
54
54
  bid = get_parent_bid(bid)
55
55
  end
@@ -94,7 +94,7 @@ module CanvasSync
94
94
  private
95
95
 
96
96
  def get_parent_hash(bid)
97
- reolve_hash(get_parent_bid(bid)).freeze
97
+ resolve_hash(get_parent_bid(bid)).freeze
98
98
  end
99
99
 
100
100
  def get_parent_bid(bid)
@@ -105,13 +105,15 @@ module CanvasSync
105
105
  if index >= 0
106
106
  @bid_stack[index]
107
107
  else
108
- pbid = Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
108
+ pbid = Batch.redis do |r|
109
+ r.hget("BID-#{bid}", "parent_bid") || r.hget("BID-#{bid}", "callback_for")
110
+ end
109
111
  @bid_stack.unshift(pbid)
110
112
  pbid
111
113
  end
112
114
  end
113
115
 
114
- def reolve_hash(bid)
116
+ def resolve_hash(bid)
115
117
  return nil unless bid.present?
116
118
  return @hash_map[bid] if @hash_map.key?(bid)
117
119
 
@@ -137,6 +139,7 @@ module CanvasSync
137
139
  end
138
140
 
139
141
  def load_all
142
+ resolve_hash(@bid_stack[0]).freeze
140
143
  while @bid_stack[0].present?
141
144
  get_parent_hash(@bid_stack[0])
142
145
  end
@@ -0,0 +1,5 @@
1
+ local r=redis.call('HGET', KEYS[1], ARGV[1])
2
+ if r == false or r < ARGV[2] then
3
+ redis.call('HINCRBY', KEYS[1], ARGV[1], 1)
4
+ end
5
+ return r or 0
@@ -0,0 +1,99 @@
1
+ require_relative './base_job'
2
+
3
+ module CanvasSync
4
+ module JobBatches
5
+ class ManagedBatchJob < BaseJob
6
+ def perform(sub_jobs, context: nil, ordered: true, concurrency: nil)
7
+ man_batch_id = SecureRandom.urlsafe_base64(10)
8
+
9
+ if concurrency == 0 || concurrency == nil || concurrency == true
10
+ concurrency = sub_jobs.count
11
+ elsif concurrency == false
12
+ concurrency = 1
13
+ end
14
+
15
+ root_batch = Batch.new
16
+
17
+ Batch.redis do |r|
18
+ r.multi do
19
+ r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
20
+ r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
21
+ r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
22
+ r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
23
+
24
+ mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
25
+ j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
26
+ j = ActiveJob::Arguments.serialize([j])
27
+ JSON.unparse(j)
28
+ end
29
+ if ordered
30
+ r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
31
+ else
32
+ r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
33
+ end
34
+ r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
35
+ end
36
+ end
37
+
38
+ root_batch.description = "Managed Batch Root (#{man_batch_id})"
39
+ root_batch.allow_context_changes = (concurrency == 1)
40
+ root_batch.context = context
41
+ root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", managed_batch_id: man_batch_id)
42
+ root_batch.jobs {}
43
+
44
+ concurrency.times do
45
+ self.class.perform_next_sequence_job(man_batch_id)
46
+ end
47
+ end
48
+
49
+ def self.cleanup_redis(status, options)
50
+ man_batch_id = options['managed_batch_id']
51
+ Batch.redis do |r|
52
+ r.del(
53
+ "MNGBID-#{man_batch_id}",
54
+ "MNGBID-#{man_batch_id}-jobs",
55
+ )
56
+ end
57
+ end
58
+
59
+ def self.job_succeeded_callback(status, options)
60
+ man_batch_id = options['managed_batch_id']
61
+ perform_next_sequence_job(man_batch_id)
62
+ end
63
+
64
+ protected
65
+
66
+ def self.perform_next_sequence_job(man_batch_id)
67
+ root_bid, ordered = Batch.redis do |r|
68
+ r.multi do
69
+ r.hget("MNGBID-#{man_batch_id}", "root_bid")
70
+ r.hget("MNGBID-#{man_batch_id}", "ordered")
71
+ end
72
+ end
73
+
74
+ next_job_json = Batch.redis do |r|
75
+ if ordered
76
+ r.lpop("MNGBID-#{man_batch_id}-jobs")
77
+ else
78
+ r.spop("MNGBID-#{man_batch_id}-jobs")
79
+ end
80
+ end
81
+
82
+ return unless next_job_json.present?
83
+
84
+ next_job = JSON.parse(next_job_json)
85
+ next_job = ActiveJob::Arguments.deserialize(next_job)[0]
86
+
87
+ Batch.new(root_bid).jobs do
88
+ Batch.new.tap do |batch|
89
+ batch.description = "Managed Batch Fiber (#{man_batch_id})"
90
+ batch.on(:success, "#{self.to_s}.job_succeeded_callback", managed_batch_id: man_batch_id)
91
+ batch.jobs do
92
+ ChainBuilder.enqueue_job(next_job)
93
+ end
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
99
+ end
@@ -4,71 +4,12 @@ module CanvasSync
4
4
  module JobBatches
5
5
  class SerialBatchJob < BaseJob
6
6
  def perform(sub_jobs, context: nil)
7
- serial_id = SecureRandom.urlsafe_base64(10)
8
-
9
- root_batch = Batch.new
10
-
11
- Batch.redis do |r|
12
- r.multi do
13
- mapped_sub_jobs = sub_jobs.map do |j|
14
- j = ActiveJob::Arguments.serialize([j])
15
- JSON.unparse(j)
16
- end
17
- r.hset("SERBID-#{serial_id}", "root_bid", root_batch.bid)
18
- r.expire("SERBID-#{serial_id}", Batch::BID_EXPIRE_TTL)
19
- r.rpush("SERBID-#{serial_id}-jobs", mapped_sub_jobs)
20
- r.expire("SERBID-#{serial_id}-jobs", Batch::BID_EXPIRE_TTL)
21
- end
22
- end
23
-
24
- root_batch.description = "Serial Batch Root (#{serial_id})"
25
- root_batch.allow_context_changes = true
26
- root_batch.context = context
27
- root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", serial_batch_id: serial_id)
28
- root_batch.jobs {}
29
-
30
- self.class.perform_next_sequence_job(serial_id)
31
- end
32
-
33
- def self.cleanup_redis(status, options)
34
- serial_id = options['serial_batch_id']
35
- Batch.redis do |r|
36
- r.del(
37
- "SERBID-#{serial_id}",
38
- "SERBID-#{serial_id}-jobs",
39
- )
40
- end
41
- end
42
-
43
- def self.job_succeeded_callback(status, options)
44
- serial_id = options['serial_batch_id']
45
- perform_next_sequence_job(serial_id)
46
- end
47
-
48
- protected
49
-
50
- def self.perform_next_sequence_job(serial_id)
51
- root_bid, next_job_json = Batch.redis do |r|
52
- r.multi do
53
- r.hget("SERBID-#{serial_id}", "root_bid")
54
- r.lpop("SERBID-#{serial_id}-jobs")
55
- end
56
- end
57
-
58
- return unless next_job_json.present?
59
-
60
- next_job = JSON.parse(next_job_json)
61
- next_job = ActiveJob::Arguments.deserialize(next_job)[0]
62
-
63
- Batch.new(root_bid).jobs do
64
- Batch.new.tap do |batch|
65
- batch.description = "Serial Batch Fiber (#{serial_id})"
66
- batch.on(:success, "#{self.to_s}.job_succeeded_callback", serial_batch_id: serial_id)
67
- batch.jobs do
68
- ChainBuilder.enqueue_job(next_job)
69
- end
70
- end
71
- end
7
+ ManagedBatchJob.new.perform(
8
+ sub_jobs,
9
+ context: context,
10
+ ordered: true,
11
+ concurrency: false,
12
+ )
72
13
  end
73
14
  end
74
15
  end
@@ -0,0 +1,193 @@
1
+ module CanvasSync
2
+ module JobBatches
3
+ class Pool
4
+ include RedisModel
5
+
6
+ HINCR_MAX = RedisScript.new(Pathname.new(__FILE__) + "../hincr_max.lua")
7
+
8
+ attr_reader :pid
9
+ redis_attr :description
10
+ redis_attr :created_at
11
+ redis_attr :concurrency, :int
12
+ redis_attr :order
13
+ redis_attr :on_failed_job, :symbol
14
+ redis_attr :clean_when_empty, :bool
15
+
16
+ def initialize(pooolid = nil, **kwargs)
17
+ if pooolid
18
+ @existing = true
19
+ @pid = pooolid
20
+ else
21
+ @pid = SecureRandom.urlsafe_base64(10)
22
+ initialize_new(**kwargs)
23
+ end
24
+ end
25
+
26
+ def self.from_pid(pid)
27
+ new(pid)
28
+ end
29
+
30
+ def <<(job_desc)
31
+ add_job(job_desc)
32
+ end
33
+
34
+ def add_job(job_desc)
35
+ add_jobs([job_desc])
36
+ end
37
+
38
+ def add_jobs(job_descs)
39
+ job_descs.each do |job_desc|
40
+ wrapper = Batch.new
41
+ wrapper.description = "Pool Job Wrapper"
42
+ checkin_event = (on_failed_job == :wait) ? :success : :complete
43
+ wrapper.on(checkin_event, "#{self.class.to_s}.job_checked_in", pool_id: pid)
44
+ wrapper.jobs {}
45
+
46
+ job_desc = job_desc.with_indifferent_access
47
+ job_desc = job_desc.merge!(
48
+ job: job_desc[:job].to_s,
49
+ pool_wrapper_batch: wrapper.bid,
50
+ )
51
+
52
+ push_job_to_pool(job_desc)
53
+ end
54
+ refill_allotment
55
+ end
56
+
57
+ def cleanup_redis
58
+ Batch.logger.debug {"Cleaning redis of pool #{pid}"}
59
+ redis do |r|
60
+ r.del(
61
+ "#{redis_key}",
62
+ "#{redis_key}-jobs",
63
+ )
64
+ end
65
+ end
66
+
67
+ def job_checked_in(status, options)
68
+ active_count = redis do |r|
69
+ r.hincrby(redis_key, "active_count", -1)
70
+ end
71
+ added_count = refill_allotment
72
+
73
+ if active_count == 0 && added_count == 0
74
+ cleanup_redis if clean_when_empty
75
+ end
76
+ end
77
+
78
+ def self.job_checked_in(status, options)
79
+ pid = options['pool_id']
80
+ from_pid(pid).job_checked_in(status, options)
81
+ end
82
+
83
+ protected
84
+
85
+ def redis_key
86
+ "POOLID-#{pid}"
87
+ end
88
+
89
+ def refill_allotment
90
+ jobs_added = 0
91
+ limit = concurrency.to_i
92
+ redis do |r|
93
+ current_count = 0
94
+ while true
95
+ current_count = HINCR_MAX.call(r, [redis_key], ["active_count", limit]).to_i
96
+ if current_count < limit
97
+ job_desc = pop_job_from_pool
98
+ if job_desc.present?
99
+ Batch.new(job_desc['pool_wrapper_batch']).jobs do
100
+ ChainBuilder.enqueue_job(job_desc)
101
+ end
102
+ jobs_added += 1
103
+ else
104
+ r.hincrby(redis_key, "active_count", -1)
105
+ break
106
+ end
107
+ else
108
+ break
109
+ end
110
+ end
111
+ r.expire(redis_key, Batch::BID_EXPIRE_TTL)
112
+ r.expire("#{redis_key}-jobs", Batch::BID_EXPIRE_TTL)
113
+ end
114
+ jobs_added
115
+ end
116
+
117
+ def push_job_to_pool(job_desc)
118
+ jobs_key = "#{redis_key}-jobs"
119
+ # This allows duplicate jobs when a Redis Set is used
120
+ job_desc['_pool_random_key_'] = SecureRandom.urlsafe_base64(10)
121
+ job_json = JSON.unparse(ActiveJob::Arguments.serialize([job_desc]))
122
+ order = self.order
123
+
124
+ redis do |r|
125
+ r.multi do
126
+ case order.to_sym
127
+ when :fifo, :lifo
128
+ r.rpush(jobs_key, job_json)
129
+ when :random
130
+ r.sadd(jobs_key, job_json)
131
+ when :priority
132
+ r.zadd(jobs_key, job_desc[:priority] || 0, job_json)
133
+ end
134
+ r.expire(jobs_key, Batch::BID_EXPIRE_TTL)
135
+ end
136
+ end
137
+ end
138
+
139
+ def pop_job_from_pool
140
+ jobs_key = "#{redis_key}-jobs"
141
+ order = self.order
142
+
143
+ job_json = nil
144
+ redis do |r|
145
+ job_json = case order.to_sym
146
+ when :fifo
147
+ r.lpop(jobs_key)
148
+ when :lifo
149
+ r.rpop(jobs_key)
150
+ when :random
151
+ r.spop(jobs_key)
152
+ when :priority
153
+ r.zpopmax(jobs_key)
154
+ end
155
+ end
156
+
157
+ return nil unless job_json.present?
158
+
159
+ ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]
160
+ end
161
+
162
+ def pending_count
163
+ order = self.order
164
+ redis do |r|
165
+ case order.to_sym
166
+ when :fifo, :lifo
167
+ r.llen(jobs_key)
168
+ when :random
169
+ r.scard(jobs_key)
170
+ when :priority
171
+ r.zcard(jobs_key)
172
+ end
173
+ end
174
+ end
175
+
176
+ def self.redis(&blk)
177
+ Batch.redis &blk
178
+ end
179
+ delegate :redis, to: :class
180
+
181
+ private
182
+
183
+ def initialize_new(concurrency: nil, order: :fifo, clean_when_empty: true, on_failed_job: :wait)
184
+ self.created_at = Time.now.utc.to_f
185
+ self.order = order
186
+ self.concurrency = concurrency
187
+ self.clean_when_empty = clean_when_empty
188
+ self.on_failed_job = on_failed_job
189
+ flush_pending_attrs
190
+ end
191
+ end
192
+ end
193
+ end
@@ -0,0 +1,69 @@
1
+ module CanvasSync
2
+ module JobBatches
3
+ module RedisModel
4
+ extend ActiveSupport::Concern
5
+
6
+ class_methods do
7
+ def redis_attr(key, type = :string, read_only: true)
8
+ class_eval <<-RUBY, __FILE__, __LINE__ + 1
9
+ def #{key}=(value)
10
+ raise "#{key} is read-only once the batch has been started" if #{read_only.to_s} && (@initialized || @existing)
11
+ @#{key} = value
12
+ if :#{type} == :json
13
+ value = JSON.unparse(value)
14
+ end
15
+ persist_bid_attr('#{key}', value)
16
+ end
17
+
18
+ def #{key}
19
+ return @#{key} if defined?(@#{key})
20
+ if (@initialized || @existing)
21
+ value = read_bid_attr('#{key}')
22
+ if :#{type} == :bool
23
+ value = value == 'true'
24
+ elsif :#{type} == :int
25
+ value = value.to_i
26
+ elsif :#{type} == :float
27
+ value = value.to_f
28
+ elsif :#{type} == :json
29
+ value = JSON.parse(value)
30
+ elsif :#{type} == :symbol
31
+ value = value&.to_sym
32
+ end
33
+ @#{key} = value
34
+ end
35
+ end
36
+ RUBY
37
+ end
38
+ end
39
+
40
+ def persist_bid_attr(attribute, value)
41
+ if @initialized || @existing
42
+ redis do |r|
43
+ r.multi do
44
+ r.hset(redis_key, attribute, value)
45
+ r.expire(redis_key, Batch::BID_EXPIRE_TTL)
46
+ end
47
+ end
48
+ else
49
+ @pending_attrs ||= {}
50
+ @pending_attrs[attribute] = value
51
+ end
52
+ end
53
+
54
+ def read_bid_attr(attribute)
55
+ redis do |r|
56
+ r.hget(redis_key, attribute)
57
+ end
58
+ end
59
+
60
+ def flush_pending_attrs
61
+ redis do |r|
62
+ r.mapped_hmset(redis_key, @pending_attrs)
63
+ end
64
+ @initialized = true
65
+ @pending_attrs = {}
66
+ end
67
+ end
68
+ end
69
+ end