canvas_sync 0.17.23.beta5 → 0.17.23

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ff0bdfef2b2ae070b727fefe7b16c20149a82e8237a922102516de8aaee508ae
4
- data.tar.gz: da0f508fe25b22b3a9913d81891cf729effaf55d0d0800001079015101be1179
3
+ metadata.gz: 1e0b64f56f9bd68aa468f8bd608000f622c07a0dbcf1a49469435788e5708545
4
+ data.tar.gz: 48bd87ac7843bf65f5e4ac11bc5271a96a032244edcb2996e710006ecf007801
5
5
  SHA512:
6
- metadata.gz: 48c84c48103000a58b3fd5803c650ccc3bf485c39d8cb09467892de0621d2607757542ed9dd275a741040231639d269978f2e5fe23d7830649470fb409bd2fd6
7
- data.tar.gz: 36cd14b22dc554ecb7110770e4cb13ce9e0ec7cc1f000ee6b567d65493f07b1c616ec0b2ac3b7234a8ba3bf633f7468a709c10ae771183ae31237ad726476826
6
+ metadata.gz: 37835e33df63bbca5a712535b1fa32574e7f6d80a186cdb1e1638b5232ccb8f96cf36c95ae10483e7e519a0ddfee7334f91cd323757302da0304d15205274a62
7
+ data.tar.gz: 785bc9a767cad6b750c23dc16f7f5fec58e88c5fb8defe7156e7931c31ee5093de25c864aa9c4b2dd6aa72f884fbbc28fcd707dfda59a8f048d551c1a8c0d361
@@ -7,14 +7,13 @@
7
7
  Rails.application.config.after_initialize do
8
8
  next unless defined?(Apartment)
9
9
 
10
- # Apartment already solves this issue (and in a better way) in newer versions
10
+ # Newer versions of Apartment already solves this issue (and in a better way)
11
11
  begin
12
12
  require('apartment/version')
13
13
  next if Gem::Version.new(Apartment::VERSION) >= Gem::Version.new('2.8.1')
14
14
  rescue LoadError
15
15
  end
16
16
 
17
-
18
17
  begin
19
18
  Rails.application.eager_load!
20
19
  ActiveRecord::Base.descendants.each do |model|
@@ -4,8 +4,6 @@ class CreateGradingPeriodGroups < ActiveRecord::Migration[5.1]
4
4
  def change
5
5
  create_table :grading_period_groups do |t|
6
6
  t.bigint :canvas_id, null: false
7
- t.bigint :canvas_course_id
8
- t.bigint :canvas_account_id
9
7
  t.string :title
10
8
  t.boolean :weighted
11
9
  t.boolean :display_totals_for_all_grading_periods
@@ -16,7 +14,5 @@ class CreateGradingPeriodGroups < ActiveRecord::Migration[5.1]
16
14
  end
17
15
 
18
16
  add_index :grading_period_groups, :canvas_id, unique: true
19
- add_index :grading_period_groups, :canvas_course_id
20
- add_index :grading_period_groups, :canvas_account_id
21
17
  end
22
18
  end
@@ -4,6 +4,4 @@ class GradingPeriodGroup < ApplicationRecord
4
4
  include CanvasSync::Record
5
5
 
6
6
  validates :canvas_id, uniqueness: true, presence: true
7
- belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true
8
- belongs_to :account, primary_key: :canvas_id, foreign_key: :canvas_account_id, optional: true
9
7
  end
@@ -76,14 +76,12 @@ module CanvasSync
76
76
  def on(event, callback, options = {})
77
77
  return unless Callback::VALID_CALLBACKS.include?(event.to_s)
78
78
  callback_key = "#{@bidkey}-callbacks-#{event}"
79
- redis do |r|
80
- r.multi do
81
- r.sadd(callback_key, JSON.unparse({
82
- callback: callback,
83
- opts: options
84
- }))
85
- r.expire(callback_key, BID_EXPIRE_TTL)
86
- end
79
+ redis.multi do |r|
80
+ r.sadd(callback_key, JSON.unparse({
81
+ callback: callback,
82
+ opts: options
83
+ }))
84
+ r.expire(callback_key, BID_EXPIRE_TTL)
87
85
  end
88
86
  end
89
87
 
@@ -93,16 +91,14 @@ module CanvasSync
93
91
  if !@existing && !@initialized
94
92
  parent_bid = Thread.current[:batch]&.bid
95
93
 
96
- redis do |r|
97
- r.multi do
98
- r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
99
- r.expire(@bidkey, BID_EXPIRE_TTL)
94
+ redis.multi do |r|
95
+ r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
96
+ r.expire(@bidkey, BID_EXPIRE_TTL)
100
97
 
101
- if parent_bid
102
- r.hincrby("BID-#{parent_bid}", "children", 1)
103
- r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
104
- r.zadd("BID-#{parent_bid}-bids", created_at, bid)
105
- end
98
+ if parent_bid
99
+ r.hincrby("BID-#{parent_bid}", "children", 1)
100
+ r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
101
+ r.zadd("BID-#{parent_bid}-bids", created_at, bid)
106
102
  end
107
103
  end
108
104
 
@@ -131,15 +127,11 @@ module CanvasSync
131
127
  end
132
128
 
133
129
  def invalidate_all
134
- redis do |r|
135
- r.setex("invalidated-bid-#{bid}", BID_EXPIRE_TTL, 1)
136
- end
130
+ redis.setex("invalidated-bid-#{bid}", BID_EXPIRE_TTL, 1)
137
131
  end
138
132
 
139
133
  def parent_bid
140
- redis do |r|
141
- r.hget(@bidkey, "parent_bid")
142
- end
134
+ redis.hget(@bidkey, "parent_bid")
143
135
  end
144
136
 
145
137
  def parent
@@ -149,7 +141,7 @@ module CanvasSync
149
141
  end
150
142
 
151
143
  def valid?(batch = self)
152
- valid = !redis { |r| r.exists?("invalidated-bid-#{batch.bid}") }
144
+ valid = !redis.exists?("invalidated-bid-#{batch.bid}")
153
145
  batch.parent ? valid && valid?(batch.parent) : valid
154
146
  end
155
147
 
@@ -162,23 +154,19 @@ module CanvasSync
162
154
  let_close!
163
155
  end
164
156
  else
165
- redis do |r|
166
- r.hset(@bidkey, 'keep_open', true)
167
- end
157
+ redis.hset(@bidkey, 'keep_open', true)
168
158
  end
169
159
  end
170
160
 
171
161
  def let_close!
172
- _, failed, pending, children, complete, success = redis do |r|
173
- r.multi do
174
- r.hset(@bidkey, 'keep_open', false)
175
-
176
- r.scard("BID-#{bid}-failed")
177
- r.hincrby("BID-#{bid}", "pending", 0)
178
- r.hincrby("BID-#{bid}", "children", 0)
179
- r.scard("BID-#{bid}-batches-complete")
180
- r.scard("BID-#{bid}-batches-success")
181
- end
162
+ _, failed, pending, children, complete, success = redis.multi do |r|
163
+ r.hset(@bidkey, 'keep_open', false)
164
+
165
+ r.scard("BID-#{bid}-failed")
166
+ r.hincrby("BID-#{bid}", "pending", 0)
167
+ r.hincrby("BID-#{bid}", "children", 0)
168
+ r.scard("BID-#{bid}-batches-complete")
169
+ r.scard("BID-#{bid}-batches-success")
182
170
  end
183
171
 
184
172
  all_success = pending.to_i.zero? && children == success
@@ -211,18 +199,14 @@ module CanvasSync
211
199
 
212
200
  def flush_pending_attrs
213
201
  super
214
- redis do |r|
215
- r.zadd("batches", created_at, bid)
216
- end
202
+ redis.zadd("batches", created_at, bid)
217
203
  end
218
204
 
219
205
  private
220
206
 
221
207
  def assert_batch_is_open
222
208
  unless defined?(@closed)
223
- redis do |r|
224
- @closed = r.hget(@bidkey, 'success') == 'true'
225
- end
209
+ @closed = redis.hget(@bidkey, 'success') == 'true'
226
210
  end
227
211
  raise "Cannot add jobs to Batch #{} bid - it has already entered the callback-stage" if @closed
228
212
  end
@@ -234,7 +218,7 @@ module CanvasSync
234
218
  redis do |r|
235
219
  tme = Time.now.utc.to_f
236
220
  added = r.zadd(@bidkey + "-jids", jids.map{|jid| [tme, jid] }, nx: true)
237
- r.multi do
221
+ r.multi do |r|
238
222
  r.hincrby(@bidkey, "pending", added)
239
223
  r.hincrby(@bidkey, "job_count", added)
240
224
  r.expire(@bidkey, BID_EXPIRE_TTL)
@@ -283,11 +267,9 @@ module CanvasSync
283
267
  end
284
268
 
285
269
  if parent_bid
286
- redis do |r|
287
- r.multi do
288
- r.sadd("BID-#{parent_bid}-dead", jid)
289
- r.expire("BID-#{parent_bid}-dead", BID_EXPIRE_TTL)
290
- end
270
+ redis.multi do
271
+ r.sadd("BID-#{parent_bid}-dead", jid)
272
+ r.expire("BID-#{parent_bid}-dead", BID_EXPIRE_TTL)
291
273
  end
292
274
  end
293
275
 
@@ -413,8 +395,21 @@ module CanvasSync
413
395
  cleanup_redis(bid)
414
396
  end
415
397
 
416
- def redis(*args, &blk)
417
- defined?(::Sidekiq) ? ::Sidekiq.redis(*args, &blk) : nil # TODO
398
+ def redis(&blk)
399
+ return RedisProxy.new unless block_given?
400
+
401
+ if Thread.current[:job_batches_redis]
402
+ yield Thread.current[:job_batches_redis]
403
+ elsif defined?(::Sidekiq)
404
+ ::Sidekiq.redis do |r|
405
+ Thread.current[:job_batches_redis] = r
406
+ yield r
407
+ ensure
408
+ Thread.current[:job_batches_redis] = nil
409
+ end
410
+ else
411
+ # TODO
412
+ end
418
413
  end
419
414
 
420
415
  def logger
@@ -433,6 +428,42 @@ module CanvasSync
433
428
  end
434
429
  end
435
430
  end
431
+
432
+ class RedisProxy
433
+ def multi(*args, &block)
434
+ Batch.redis do |r|
435
+ if block.arity == 1
436
+ r.multi(*args) do
437
+ block.call(r)
438
+ end
439
+ else
440
+ r.multi(*args, &block)
441
+ end
442
+ end
443
+ end
444
+
445
+ def pipelined(*args, &block)
446
+ Batch.redis do |r|
447
+ if block.arity == 1
448
+ r.pipelined(*args) do
449
+ block.call(r)
450
+ end
451
+ else
452
+ r.pipelined(*args, &block)
453
+ end
454
+ end
455
+ end
456
+
457
+ def method_missing(method_name, *arguments, &block)
458
+ Batch.redis do |r|
459
+ r.send(method_name, *arguments, &block)
460
+ end
461
+ end
462
+
463
+ def respond_to_missing?(method_name, include_private = false)
464
+ super || Redis.method_defined?(method_name)
465
+ end
466
+ end
436
467
  end
437
468
 
438
469
  ActiveJob::Base.include BatchAwareJob
@@ -64,18 +64,14 @@ module CanvasSync
64
64
  let_close!
65
65
  end
66
66
  else
67
- redis do |r|
68
- r.hset(redis_key, 'keep_open', true)
69
- end
67
+ redis.hset(redis_key, 'keep_open', true)
70
68
  end
71
69
  end
72
70
 
73
71
  def let_close!
74
- _, active_count = redis do |r|
75
- r.multi do
76
- r.hset(redis_key, 'keep_open', false)
77
- r.hincrby(redis_key, "active_count", 0)
78
- end
72
+ _, active_count = redis.multi do |r|
73
+ r.hset(redis_key, 'keep_open', false)
74
+ r.hincrby(redis_key, "active_count", 0)
79
75
  end
80
76
 
81
77
  if active_count == 0 && pending_count == 0
@@ -95,35 +91,38 @@ module CanvasSync
95
91
  end
96
92
 
97
93
  def active_count
98
- redis do |r|
99
- r.hincrby(redis_key, "active_count", 0)
100
- end
94
+ redis.hincrby(redis_key, "active_count", 0)
101
95
  end
102
96
 
103
97
  def pending_count
104
98
  jobs_key = "#{redis_key}-jobs"
105
99
  order = self.order || 'fifo'
106
- redis do |r|
107
- case order.to_sym
108
- when :fifo, :lifo
109
- r.llen(jobs_key)
110
- when :random
111
- r.scard(jobs_key)
112
- when :priority
113
- r.zcard(jobs_key)
114
- end
100
+ case order.to_sym
101
+ when :fifo, :lifo
102
+ redis.llen(jobs_key)
103
+ when :random
104
+ redis.scard(jobs_key)
105
+ when :priority
106
+ redis.zcard(jobs_key)
115
107
  end
116
108
  end
117
109
 
118
110
  def job_checked_in(status, options)
119
- active_count = redis do |r|
111
+ active_count, pending_count = redis do |r|
120
112
  return unless r.exists?(redis_key)
121
- r.hincrby(redis_key, "active_count", -1)
113
+
114
+ # Make sure this is loaded outside of the pipeline
115
+ self.order
116
+
117
+ redis.multi do
118
+ r.hincrby(redis_key, "active_count", -1)
119
+ self.pending_count
120
+ end
122
121
  end
123
122
 
124
123
  added_count = refill_allotment
125
- if active_count == 0 && added_count == 0
126
- if clean_when_empty && redis {|r| r.hget(redis_key, 'keep_open') } != 'true'
124
+ if active_count == 0 && added_count == 0 && pending_count == 0
125
+ if clean_when_empty && redis.hget(redis_key, 'keep_open') != 'true'
127
126
  cleanup_redis
128
127
  end
129
128
  end
@@ -175,18 +174,16 @@ module CanvasSync
175
174
  job_json = JSON.unparse(ActiveJob::Arguments.serialize([job_desc]))
176
175
  order = self.order
177
176
 
178
- redis do |r|
179
- r.multi do
180
- case order.to_sym
181
- when :fifo, :lifo
182
- r.rpush(jobs_key, job_json)
183
- when :random
184
- r.sadd(jobs_key, job_json)
185
- when :priority
186
- r.zadd(jobs_key, job_desc[:priority] || 0, job_json)
187
- end
188
- r.expire(jobs_key, Batch::BID_EXPIRE_TTL)
177
+ redis.multi do |r|
178
+ case order.to_sym
179
+ when :fifo, :lifo
180
+ r.rpush(jobs_key, job_json)
181
+ when :random
182
+ r.sadd(jobs_key, job_json)
183
+ when :priority
184
+ r.zadd(jobs_key, job_desc[:priority] || 0, job_json)
189
185
  end
186
+ r.expire(jobs_key, Batch::BID_EXPIRE_TTL)
190
187
  end
191
188
  end
192
189
 
@@ -194,18 +191,15 @@ module CanvasSync
194
191
  jobs_key = "#{redis_key}-jobs"
195
192
  order = self.order
196
193
 
197
- job_json = nil
198
- redis do |r|
199
- job_json = case order.to_sym
200
- when :fifo
201
- r.lpop(jobs_key)
202
- when :lifo
203
- r.rpop(jobs_key)
204
- when :random
205
- r.spop(jobs_key)
206
- when :priority
207
- r.zpopmax(jobs_key)
208
- end
194
+ job_json = case order.to_sym
195
+ when :fifo
196
+ redis.lpop(jobs_key)
197
+ when :lifo
198
+ redis.rpop(jobs_key)
199
+ when :random
200
+ redis.spop(jobs_key)
201
+ when :priority
202
+ redis.zpopmax(jobs_key)
209
203
  end
210
204
 
211
205
  return nil unless job_json.present?
@@ -218,13 +212,9 @@ module CanvasSync
218
212
  end
219
213
  delegate :redis, to: :class
220
214
 
221
- protected
222
-
223
215
  def flush_pending_attrs
224
216
  super
225
- redis do |r|
226
- r.zadd("pools", created_at, pid)
227
- end
217
+ redis.zadd("pools", created_at, pid)
228
218
  end
229
219
 
230
220
  private
@@ -9,7 +9,7 @@ module CanvasSync
9
9
  # will be started for each term in that scope. :models should be an array of
10
10
  # models to sync.
11
11
  def perform(options)
12
- CanvasSync.get_canvas_sync_client(batch_context).terms("self").all_pages!.each do |term_params|
12
+ CanvasSync.get_canvas_sync_client(batch_context).terms("self", workflow_state: ['all']).all_pages!.each do |term_params|
13
13
  if account_id = batch_context[:account_id]
14
14
  # These branches are primarily to support Legacy apps
15
15
  if Term.respond_to?(:create_or_update) && Term.method(:create_or_update).arity.abs == 2
@@ -457,12 +457,6 @@ grading_period_groups:
457
457
  grading_period_group_id:
458
458
  database_column_name: canvas_id
459
459
  type: integer
460
- canvas_course_id:
461
- database_column_name: canvas_course_id
462
- type: integer
463
- canvas_account_id:
464
- database_column_name: canvas_account_id
465
- type: integer
466
460
  title:
467
461
  database_column_name: title
468
462
  type: string
@@ -1,3 +1,3 @@
1
1
  module CanvasSync
2
- VERSION = "0.17.23.beta5".freeze
2
+ VERSION = "0.17.23".freeze
3
3
  end
@@ -10,6 +10,4 @@ class GradingPeriodGroup < ApplicationRecord
10
10
  include CanvasSync::Record
11
11
 
12
12
  validates :canvas_id, uniqueness: true, presence: true
13
- belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true
14
- belongs_to :account, primary_key: :canvas_id, foreign_key: :canvas_account_id, optional: true
15
13
  end
@@ -12,6 +12,7 @@ class Term < ApplicationRecord
12
12
 
13
13
  validates :canvas_id, uniqueness: true, presence: true
14
14
  has_many :courses, foreign_key: :canvas_term_id, primary_key: :canvas_id
15
+ belongs_to :grading_period_group, primary_key: :canvas_id, foreign_key: :grading_period_group_id, optional: true
15
16
 
16
17
  api_syncable({
17
18
  canvas_id: :id,
@@ -10,8 +10,6 @@ class CreateGradingPeriodGroups < ActiveRecord::Migration[5.1]
10
10
  def change
11
11
  create_table :grading_period_groups do |t|
12
12
  t.bigint :canvas_id, null: false
13
- t.bigint :canvas_course_id
14
- t.bigint :canvas_account_id
15
13
  t.string :title
16
14
  t.boolean :weighted
17
15
  t.boolean :display_totals_for_all_grading_periods
@@ -22,7 +20,5 @@ class CreateGradingPeriodGroups < ActiveRecord::Migration[5.1]
22
20
  end
23
21
 
24
22
  add_index :grading_period_groups, :canvas_id, unique: true
25
- add_index :grading_period_groups, :canvas_course_id
26
- add_index :grading_period_groups, :canvas_account_id
27
23
  end
28
24
  end
@@ -10,7 +10,7 @@
10
10
  #
11
11
  # It's strongly recommended that you check this file into your version control system.
12
12
 
13
- ActiveRecord::Schema.define(version: 2021_09_07_233331) do
13
+ ActiveRecord::Schema.define(version: 2021_10_01_184920) do
14
14
 
15
15
  # These are extensions that must be enabled in order to support this database
16
16
  enable_extension "plpgsql"
@@ -175,16 +175,12 @@ ActiveRecord::Schema.define(version: 2021_09_07_233331) do
175
175
 
176
176
  create_table "grading_period_groups", force: :cascade do |t|
177
177
  t.bigint "canvas_id", null: false
178
- t.bigint "canvas_course_id"
179
- t.bigint "canvas_account_id"
180
178
  t.string "title"
181
179
  t.boolean "weighted"
182
180
  t.boolean "display_totals_for_all_grading_periods"
183
181
  t.string "workflow_state"
184
182
  t.datetime "created_at", null: false
185
183
  t.datetime "updated_at", null: false
186
- t.index ["canvas_account_id"], name: "index_grading_period_groups_on_canvas_account_id"
187
- t.index ["canvas_course_id"], name: "index_grading_period_groups_on_canvas_course_id"
188
184
  t.index ["canvas_id"], name: "index_grading_period_groups_on_canvas_id", unique: true
189
185
  end
190
186