canvas_sync 0.17.0 → 0.17.3.beta3

Sign up to get free protection for your applications and to get access to all the features.
@@ -17,7 +17,7 @@ module CanvasSync
17
17
  def local_bid
18
18
  bid = @bid_stack[-1]
19
19
  while bid.present?
20
- bhash = reolve_hash(bid)
20
+ bhash = resolve_hash(bid)
21
21
  return bid if bhash
22
22
  bid = get_parent_bid(bid)
23
23
  end
@@ -49,7 +49,7 @@ module CanvasSync
49
49
  def [](key)
50
50
  bid = @bid_stack[-1]
51
51
  while bid.present?
52
- bhash = reolve_hash(bid)
52
+ bhash = resolve_hash(bid)
53
53
  return bhash[key] if bhash&.key?(key)
54
54
  bid = get_parent_bid(bid)
55
55
  end
@@ -94,7 +94,7 @@ module CanvasSync
94
94
  private
95
95
 
96
96
  def get_parent_hash(bid)
97
- reolve_hash(get_parent_bid(bid)).freeze
97
+ resolve_hash(get_parent_bid(bid)).freeze
98
98
  end
99
99
 
100
100
  def get_parent_bid(bid)
@@ -105,13 +105,15 @@ module CanvasSync
105
105
  if index >= 0
106
106
  @bid_stack[index]
107
107
  else
108
- pbid = Batch.redis { |r| r.hget("BID-#{bid}", "parent_bid") }
108
+ pbid = Batch.redis do |r|
109
+ r.hget("BID-#{bid}", "parent_bid") || r.hget("BID-#{bid}", "callback_for")
110
+ end
109
111
  @bid_stack.unshift(pbid)
110
112
  pbid
111
113
  end
112
114
  end
113
115
 
114
- def reolve_hash(bid)
116
+ def resolve_hash(bid)
115
117
  return nil unless bid.present?
116
118
  return @hash_map[bid] if @hash_map.key?(bid)
117
119
 
@@ -137,6 +139,7 @@ module CanvasSync
137
139
  end
138
140
 
139
141
  def load_all
142
+ resolve_hash(@bid_stack[0]).freeze
140
143
  while @bid_stack[0].present?
141
144
  get_parent_hash(@bid_stack[0])
142
145
  end
@@ -0,0 +1,5 @@
1
+ local r=redis.call('HGET', KEYS[1], ARGV[1])
2
+ if r == false or r < ARGV[2] then
3
+ redis.call('HINCRBY', KEYS[1], ARGV[1], 1)
4
+ end
5
+ return r or 0
@@ -0,0 +1,99 @@
1
+ require_relative './base_job'
2
+
3
+ module CanvasSync
4
+ module JobBatches
5
+ class ManagedBatchJob < BaseJob
6
+ def perform(sub_jobs, context: nil, ordered: true, concurrency: nil)
7
+ man_batch_id = SecureRandom.urlsafe_base64(10)
8
+
9
+ if concurrency == 0 || concurrency == nil || concurrency == true
10
+ concurrency = sub_jobs.count
11
+ elsif concurrency == false
12
+ concurrency = 1
13
+ end
14
+
15
+ root_batch = Batch.new
16
+
17
+ Batch.redis do |r|
18
+ r.multi do
19
+ r.hset("MNGBID-#{man_batch_id}", "root_bid", root_batch.bid)
20
+ r.hset("MNGBID-#{man_batch_id}", "ordered", ordered)
21
+ r.hset("MNGBID-#{man_batch_id}", "concurrency", concurrency)
22
+ r.expire("MNGBID-#{man_batch_id}", Batch::BID_EXPIRE_TTL)
23
+
24
+ mapped_sub_jobs = sub_jobs.each_with_index.map do |j, i|
25
+ j['_mngbid_index_'] = i # This allows duplicate jobs when a Redis Set is used
26
+ j = ActiveJob::Arguments.serialize([j])
27
+ JSON.unparse(j)
28
+ end
29
+ if ordered
30
+ r.rpush("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
31
+ else
32
+ r.sadd("MNGBID-#{man_batch_id}-jobs", mapped_sub_jobs)
33
+ end
34
+ r.expire("MNGBID-#{man_batch_id}-jobs", Batch::BID_EXPIRE_TTL)
35
+ end
36
+ end
37
+
38
+ root_batch.description = "Managed Batch Root (#{man_batch_id})"
39
+ root_batch.allow_context_changes = (concurrency == 1)
40
+ root_batch.context = context
41
+ root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", managed_batch_id: man_batch_id)
42
+ root_batch.jobs {}
43
+
44
+ concurrency.times do
45
+ self.class.perform_next_sequence_job(man_batch_id)
46
+ end
47
+ end
48
+
49
+ def self.cleanup_redis(status, options)
50
+ man_batch_id = options['managed_batch_id']
51
+ Batch.redis do |r|
52
+ r.del(
53
+ "MNGBID-#{man_batch_id}",
54
+ "MNGBID-#{man_batch_id}-jobs",
55
+ )
56
+ end
57
+ end
58
+
59
+ def self.job_succeeded_callback(status, options)
60
+ man_batch_id = options['managed_batch_id']
61
+ perform_next_sequence_job(man_batch_id)
62
+ end
63
+
64
+ protected
65
+
66
+ def self.perform_next_sequence_job(man_batch_id)
67
+ root_bid, ordered = Batch.redis do |r|
68
+ r.multi do
69
+ r.hget("MNGBID-#{man_batch_id}", "root_bid")
70
+ r.hget("MNGBID-#{man_batch_id}", "ordered")
71
+ end
72
+ end
73
+
74
+ next_job_json = Batch.redis do |r|
75
+ if ordered
76
+ r.lpop("MNGBID-#{man_batch_id}-jobs")
77
+ else
78
+ r.spop("MNGBID-#{man_batch_id}-jobs")
79
+ end
80
+ end
81
+
82
+ return unless next_job_json.present?
83
+
84
+ next_job = JSON.parse(next_job_json)
85
+ next_job = ActiveJob::Arguments.deserialize(next_job)[0]
86
+
87
+ Batch.new(root_bid).jobs do
88
+ Batch.new.tap do |batch|
89
+ batch.description = "Managed Batch Fiber (#{man_batch_id})"
90
+ batch.on(:success, "#{self.to_s}.job_succeeded_callback", managed_batch_id: man_batch_id)
91
+ batch.jobs do
92
+ ChainBuilder.enqueue_job(next_job)
93
+ end
94
+ end
95
+ end
96
+ end
97
+ end
98
+ end
99
+ end
@@ -4,71 +4,12 @@ module CanvasSync
4
4
  module JobBatches
5
5
  class SerialBatchJob < BaseJob
6
6
  def perform(sub_jobs, context: nil)
7
- serial_id = SecureRandom.urlsafe_base64(10)
8
-
9
- root_batch = Batch.new
10
-
11
- Batch.redis do |r|
12
- r.multi do
13
- mapped_sub_jobs = sub_jobs.map do |j|
14
- j = ActiveJob::Arguments.serialize([j])
15
- JSON.unparse(j)
16
- end
17
- r.hset("SERBID-#{serial_id}", "root_bid", root_batch.bid)
18
- r.expire("SERBID-#{serial_id}", Batch::BID_EXPIRE_TTL)
19
- r.rpush("SERBID-#{serial_id}-jobs", mapped_sub_jobs)
20
- r.expire("SERBID-#{serial_id}-jobs", Batch::BID_EXPIRE_TTL)
21
- end
22
- end
23
-
24
- root_batch.description = "Serial Batch Root (#{serial_id})"
25
- root_batch.allow_context_changes = true
26
- root_batch.context = context
27
- root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", serial_batch_id: serial_id)
28
- root_batch.jobs {}
29
-
30
- self.class.perform_next_sequence_job(serial_id)
31
- end
32
-
33
- def self.cleanup_redis(status, options)
34
- serial_id = options['serial_batch_id']
35
- Batch.redis do |r|
36
- r.del(
37
- "SERBID-#{serial_id}",
38
- "SERBID-#{serial_id}-jobs",
39
- )
40
- end
41
- end
42
-
43
- def self.job_succeeded_callback(status, options)
44
- serial_id = options['serial_batch_id']
45
- perform_next_sequence_job(serial_id)
46
- end
47
-
48
- protected
49
-
50
- def self.perform_next_sequence_job(serial_id)
51
- root_bid, next_job_json = Batch.redis do |r|
52
- r.multi do
53
- r.hget("SERBID-#{serial_id}", "root_bid")
54
- r.lpop("SERBID-#{serial_id}-jobs")
55
- end
56
- end
57
-
58
- return unless next_job_json.present?
59
-
60
- next_job = JSON.parse(next_job_json)
61
- next_job = ActiveJob::Arguments.deserialize(next_job)[0]
62
-
63
- Batch.new(root_bid).jobs do
64
- Batch.new.tap do |batch|
65
- batch.description = "Serial Batch Fiber (#{serial_id})"
66
- batch.on(:success, "#{self.to_s}.job_succeeded_callback", serial_batch_id: serial_id)
67
- batch.jobs do
68
- ChainBuilder.enqueue_job(next_job)
69
- end
70
- end
71
- end
7
+ ManagedBatchJob.new.perform(
8
+ sub_jobs,
9
+ context: context,
10
+ ordered: true,
11
+ concurrency: false,
12
+ )
72
13
  end
73
14
  end
74
15
  end
@@ -0,0 +1,193 @@
1
+ module CanvasSync
2
+ module JobBatches
3
+ class Pool
4
+ include RedisModel
5
+
6
+ HINCR_MAX = RedisScript.new(Pathname.new(__FILE__) + "../hincr_max.lua")
7
+
8
+ attr_reader :pid
9
+ redis_attr :description
10
+ redis_attr :created_at
11
+ redis_attr :concurrency, :int
12
+ redis_attr :order
13
+ redis_attr :on_failed_job, :symbol
14
+ redis_attr :clean_when_empty, :bool
15
+
16
+ def initialize(pooolid = nil, **kwargs)
17
+ if pooolid
18
+ @existing = true
19
+ @pid = pooolid
20
+ else
21
+ @pid = SecureRandom.urlsafe_base64(10)
22
+ initialize_new(**kwargs)
23
+ end
24
+ end
25
+
26
+ def self.from_pid(pid)
27
+ new(pid)
28
+ end
29
+
30
+ def <<(job_desc)
31
+ add_job(job_desc)
32
+ end
33
+
34
+ def add_job(job_desc)
35
+ add_jobs([job_desc])
36
+ end
37
+
38
+ def add_jobs(job_descs)
39
+ job_descs.each do |job_desc|
40
+ wrapper = Batch.new
41
+ wrapper.description = "Pool Job Wrapper"
42
+ checkin_event = (on_failed_job == :wait) ? :success : :complete
43
+ wrapper.on(checkin_event, "#{self.class.to_s}.job_checked_in", pool_id: pid)
44
+ wrapper.jobs {}
45
+
46
+ job_desc = job_desc.with_indifferent_access
47
+ job_desc = job_desc.merge!(
48
+ job: job_desc[:job].to_s,
49
+ pool_wrapper_batch: wrapper.bid,
50
+ )
51
+
52
+ push_job_to_pool(job_desc)
53
+ end
54
+ refill_allotment
55
+ end
56
+
57
+ def cleanup_redis
58
+ Batch.logger.debug {"Cleaning redis of pool #{pid}"}
59
+ redis do |r|
60
+ r.del(
61
+ "#{redis_key}",
62
+ "#{redis_key}-jobs",
63
+ )
64
+ end
65
+ end
66
+
67
+ def job_checked_in(status, options)
68
+ active_count = redis do |r|
69
+ r.hincrby(redis_key, "active_count", -1)
70
+ end
71
+ added_count = refill_allotment
72
+
73
+ if active_count == 0 && added_count == 0
74
+ cleanup_redis if clean_when_empty
75
+ end
76
+ end
77
+
78
+ def self.job_checked_in(status, options)
79
+ pid = options['pool_id']
80
+ from_pid(pid).job_checked_in(status, options)
81
+ end
82
+
83
+ protected
84
+
85
+ def redis_key
86
+ "POOLID-#{pid}"
87
+ end
88
+
89
+ def refill_allotment
90
+ jobs_added = 0
91
+ limit = concurrency.to_i
92
+ redis do |r|
93
+ current_count = 0
94
+ while true
95
+ current_count = HINCR_MAX.call(r, [redis_key], ["active_count", limit]).to_i
96
+ if current_count < limit
97
+ job_desc = pop_job_from_pool
98
+ if job_desc.present?
99
+ Batch.new(job_desc['pool_wrapper_batch']).jobs do
100
+ ChainBuilder.enqueue_job(job_desc)
101
+ end
102
+ jobs_added += 1
103
+ else
104
+ r.hincrby(redis_key, "active_count", -1)
105
+ break
106
+ end
107
+ else
108
+ break
109
+ end
110
+ end
111
+ r.expire(redis_key, Batch::BID_EXPIRE_TTL)
112
+ r.expire("#{redis_key}-jobs", Batch::BID_EXPIRE_TTL)
113
+ end
114
+ jobs_added
115
+ end
116
+
117
+ def push_job_to_pool(job_desc)
118
+ jobs_key = "#{redis_key}-jobs"
119
+ # This allows duplicate jobs when a Redis Set is used
120
+ job_desc['_pool_random_key_'] = SecureRandom.urlsafe_base64(10)
121
+ job_json = JSON.unparse(ActiveJob::Arguments.serialize([job_desc]))
122
+ order = self.order
123
+
124
+ redis do |r|
125
+ r.multi do
126
+ case order.to_sym
127
+ when :fifo, :lifo
128
+ r.rpush(jobs_key, job_json)
129
+ when :random
130
+ r.sadd(jobs_key, job_json)
131
+ when :priority
132
+ r.zadd(jobs_key, job_desc[:priority] || 0, job_json)
133
+ end
134
+ r.expire(jobs_key, Batch::BID_EXPIRE_TTL)
135
+ end
136
+ end
137
+ end
138
+
139
+ def pop_job_from_pool
140
+ jobs_key = "#{redis_key}-jobs"
141
+ order = self.order
142
+
143
+ job_json = nil
144
+ redis do |r|
145
+ job_json = case order.to_sym
146
+ when :fifo
147
+ r.lpop(jobs_key)
148
+ when :lifo
149
+ r.rpop(jobs_key)
150
+ when :random
151
+ r.spop(jobs_key)
152
+ when :priority
153
+ r.zpopmax(jobs_key)
154
+ end
155
+ end
156
+
157
+ return nil unless job_json.present?
158
+
159
+ ActiveJob::Arguments.deserialize(JSON.parse(job_json))[0]
160
+ end
161
+
162
+ def pending_count
163
+ order = self.order
164
+ redis do |r|
165
+ case order.to_sym
166
+ when :fifo, :lifo
167
+ r.llen(jobs_key)
168
+ when :random
169
+ r.scard(jobs_key)
170
+ when :priority
171
+ r.zcard(jobs_key)
172
+ end
173
+ end
174
+ end
175
+
176
+ def self.redis(&blk)
177
+ Batch.redis &blk
178
+ end
179
+ delegate :redis, to: :class
180
+
181
+ private
182
+
183
+ def initialize_new(concurrency: nil, order: :fifo, clean_when_empty: true, on_failed_job: :wait)
184
+ self.created_at = Time.now.utc.to_f
185
+ self.order = order
186
+ self.concurrency = concurrency
187
+ self.clean_when_empty = clean_when_empty
188
+ self.on_failed_job = on_failed_job
189
+ flush_pending_attrs
190
+ end
191
+ end
192
+ end
193
+ end
@@ -0,0 +1,69 @@
1
+ module CanvasSync
2
+ module JobBatches
3
+ module RedisModel
4
+ extend ActiveSupport::Concern
5
+
6
+ class_methods do
7
+ def redis_attr(key, type = :string, read_only: true)
8
+ class_eval <<-RUBY, __FILE__, __LINE__ + 1
9
+ def #{key}=(value)
10
+ raise "#{key} is read-only once the batch has been started" if #{read_only.to_s} && (@initialized || @existing)
11
+ @#{key} = value
12
+ if :#{type} == :json
13
+ value = JSON.unparse(value)
14
+ end
15
+ persist_bid_attr('#{key}', value)
16
+ end
17
+
18
+ def #{key}
19
+ return @#{key} if defined?(@#{key})
20
+ if (@initialized || @existing)
21
+ value = read_bid_attr('#{key}')
22
+ if :#{type} == :bool
23
+ value = value == 'true'
24
+ elsif :#{type} == :int
25
+ value = value.to_i
26
+ elsif :#{type} == :float
27
+ value = value.to_f
28
+ elsif :#{type} == :json
29
+ value = JSON.parse(value)
30
+ elsif :#{type} == :symbol
31
+ value = value&.to_sym
32
+ end
33
+ @#{key} = value
34
+ end
35
+ end
36
+ RUBY
37
+ end
38
+ end
39
+
40
+ def persist_bid_attr(attribute, value)
41
+ if @initialized || @existing
42
+ redis do |r|
43
+ r.multi do
44
+ r.hset(redis_key, attribute, value)
45
+ r.expire(redis_key, Batch::BID_EXPIRE_TTL)
46
+ end
47
+ end
48
+ else
49
+ @pending_attrs ||= {}
50
+ @pending_attrs[attribute] = value
51
+ end
52
+ end
53
+
54
+ def read_bid_attr(attribute)
55
+ redis do |r|
56
+ r.hget(redis_key, attribute)
57
+ end
58
+ end
59
+
60
+ def flush_pending_attrs
61
+ redis do |r|
62
+ r.mapped_hmset(redis_key, @pending_attrs)
63
+ end
64
+ @initialized = true
65
+ @pending_attrs = {}
66
+ end
67
+ end
68
+ end
69
+ end