canvas_sync 0.17.23.beta5 → 0.17.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ff0bdfef2b2ae070b727fefe7b16c20149a82e8237a922102516de8aaee508ae
4
- data.tar.gz: da0f508fe25b22b3a9913d81891cf729effaf55d0d0800001079015101be1179
3
+ metadata.gz: 1e0b64f56f9bd68aa468f8bd608000f622c07a0dbcf1a49469435788e5708545
4
+ data.tar.gz: 48bd87ac7843bf65f5e4ac11bc5271a96a032244edcb2996e710006ecf007801
5
5
  SHA512:
6
- metadata.gz: 48c84c48103000a58b3fd5803c650ccc3bf485c39d8cb09467892de0621d2607757542ed9dd275a741040231639d269978f2e5fe23d7830649470fb409bd2fd6
7
- data.tar.gz: 36cd14b22dc554ecb7110770e4cb13ce9e0ec7cc1f000ee6b567d65493f07b1c616ec0b2ac3b7234a8ba3bf633f7468a709c10ae771183ae31237ad726476826
6
+ metadata.gz: 37835e33df63bbca5a712535b1fa32574e7f6d80a186cdb1e1638b5232ccb8f96cf36c95ae10483e7e519a0ddfee7334f91cd323757302da0304d15205274a62
7
+ data.tar.gz: 785bc9a767cad6b750c23dc16f7f5fec58e88c5fb8defe7156e7931c31ee5093de25c864aa9c4b2dd6aa72f884fbbc28fcd707dfda59a8f048d551c1a8c0d361
@@ -7,14 +7,13 @@
7
7
  Rails.application.config.after_initialize do
8
8
  next unless defined?(Apartment)
9
9
 
10
- # Apartment already solves this issue (and in a better way) in newer versions
10
+ # Newer versions of Apartment already solves this issue (and in a better way)
11
11
  begin
12
12
  require('apartment/version')
13
13
  next if Gem::Version.new(Apartment::VERSION) >= Gem::Version.new('2.8.1')
14
14
  rescue LoadError
15
15
  end
16
16
 
17
-
18
17
  begin
19
18
  Rails.application.eager_load!
20
19
  ActiveRecord::Base.descendants.each do |model|
@@ -4,8 +4,6 @@ class CreateGradingPeriodGroups < ActiveRecord::Migration[5.1]
4
4
  def change
5
5
  create_table :grading_period_groups do |t|
6
6
  t.bigint :canvas_id, null: false
7
- t.bigint :canvas_course_id
8
- t.bigint :canvas_account_id
9
7
  t.string :title
10
8
  t.boolean :weighted
11
9
  t.boolean :display_totals_for_all_grading_periods
@@ -16,7 +14,5 @@ class CreateGradingPeriodGroups < ActiveRecord::Migration[5.1]
16
14
  end
17
15
 
18
16
  add_index :grading_period_groups, :canvas_id, unique: true
19
- add_index :grading_period_groups, :canvas_course_id
20
- add_index :grading_period_groups, :canvas_account_id
21
17
  end
22
18
  end
@@ -4,6 +4,4 @@ class GradingPeriodGroup < ApplicationRecord
4
4
  include CanvasSync::Record
5
5
 
6
6
  validates :canvas_id, uniqueness: true, presence: true
7
- belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true
8
- belongs_to :account, primary_key: :canvas_id, foreign_key: :canvas_account_id, optional: true
9
7
  end
@@ -76,14 +76,12 @@ module CanvasSync
76
76
  def on(event, callback, options = {})
77
77
  return unless Callback::VALID_CALLBACKS.include?(event.to_s)
78
78
  callback_key = "#{@bidkey}-callbacks-#{event}"
79
- redis do |r|
80
- r.multi do
81
- r.sadd(callback_key, JSON.unparse({
82
- callback: callback,
83
- opts: options
84
- }))
85
- r.expire(callback_key, BID_EXPIRE_TTL)
86
- end
79
+ redis.multi do |r|
80
+ r.sadd(callback_key, JSON.unparse({
81
+ callback: callback,
82
+ opts: options
83
+ }))
84
+ r.expire(callback_key, BID_EXPIRE_TTL)
87
85
  end
88
86
  end
89
87
 
@@ -93,16 +91,14 @@ module CanvasSync
93
91
  if !@existing && !@initialized
94
92
  parent_bid = Thread.current[:batch]&.bid
95
93
 
96
- redis do |r|
97
- r.multi do
98
- r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
99
- r.expire(@bidkey, BID_EXPIRE_TTL)
94
+ redis.multi do |r|
95
+ r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
96
+ r.expire(@bidkey, BID_EXPIRE_TTL)
100
97
 
101
- if parent_bid
102
- r.hincrby("BID-#{parent_bid}", "children", 1)
103
- r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
104
- r.zadd("BID-#{parent_bid}-bids", created_at, bid)
105
- end
98
+ if parent_bid
99
+ r.hincrby("BID-#{parent_bid}", "children", 1)
100
+ r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
101
+ r.zadd("BID-#{parent_bid}-bids", created_at, bid)
106
102
  end
107
103
  end
108
104
 
@@ -131,15 +127,11 @@ module CanvasSync
131
127
  end
132
128
 
133
129
  def invalidate_all
134
- redis do |r|
135
- r.setex("invalidated-bid-#{bid}", BID_EXPIRE_TTL, 1)
136
- end
130
+ redis.setex("invalidated-bid-#{bid}", BID_EXPIRE_TTL, 1)
137
131
  end
138
132
 
139
133
  def parent_bid
140
- redis do |r|
141
- r.hget(@bidkey, "parent_bid")
142
- end
134
+ redis.hget(@bidkey, "parent_bid")
143
135
  end
144
136
 
145
137
  def parent
@@ -149,7 +141,7 @@ module CanvasSync
149
141
  end
150
142
 
151
143
  def valid?(batch = self)
152
- valid = !redis { |r| r.exists?("invalidated-bid-#{batch.bid}") }
144
+ valid = !redis.exists?("invalidated-bid-#{batch.bid}")
153
145
  batch.parent ? valid && valid?(batch.parent) : valid
154
146
  end
155
147
 
@@ -162,23 +154,19 @@ module CanvasSync
162
154
  let_close!
163
155
  end
164
156
  else
165
- redis do |r|
166
- r.hset(@bidkey, 'keep_open', true)
167
- end
157
+ redis.hset(@bidkey, 'keep_open', true)
168
158
  end
169
159
  end
170
160
 
171
161
  def let_close!
172
- _, failed, pending, children, complete, success = redis do |r|
173
- r.multi do
174
- r.hset(@bidkey, 'keep_open', false)
175
-
176
- r.scard("BID-#{bid}-failed")
177
- r.hincrby("BID-#{bid}", "pending", 0)
178
- r.hincrby("BID-#{bid}", "children", 0)
179
- r.scard("BID-#{bid}-batches-complete")
180
- r.scard("BID-#{bid}-batches-success")
181
- end
162
+ _, failed, pending, children, complete, success = redis.multi do |r|
163
+ r.hset(@bidkey, 'keep_open', false)
164
+
165
+ r.scard("BID-#{bid}-failed")
166
+ r.hincrby("BID-#{bid}", "pending", 0)
167
+ r.hincrby("BID-#{bid}", "children", 0)
168
+ r.scard("BID-#{bid}-batches-complete")
169
+ r.scard("BID-#{bid}-batches-success")
182
170
  end
183
171
 
184
172
  all_success = pending.to_i.zero? && children == success
@@ -211,18 +199,14 @@ module CanvasSync
211
199
 
212
200
  def flush_pending_attrs
213
201
  super
214
- redis do |r|
215
- r.zadd("batches", created_at, bid)
216
- end
202
+ redis.zadd("batches", created_at, bid)
217
203
  end
218
204
 
219
205
  private
220
206
 
221
207
  def assert_batch_is_open
222
208
  unless defined?(@closed)
223
- redis do |r|
224
- @closed = r.hget(@bidkey, 'success') == 'true'
225
- end
209
+ @closed = redis.hget(@bidkey, 'success') == 'true'
226
210
  end
227
211
  raise "Cannot add jobs to Batch #{} bid - it has already entered the callback-stage" if @closed
228
212
  end
@@ -234,7 +218,7 @@ module CanvasSync
234
218
  redis do |r|
235
219
  tme = Time.now.utc.to_f
236
220
  added = r.zadd(@bidkey + "-jids", jids.map{|jid| [tme, jid] }, nx: true)
237
- r.multi do
221
+ r.multi do |r|
238
222
  r.hincrby(@bidkey, "pending", added)
239
223
  r.hincrby(@bidkey, "job_count", added)
240
224
  r.expire(@bidkey, BID_EXPIRE_TTL)
@@ -283,11 +267,9 @@ module CanvasSync
283
267
  end
284
268
 
285
269
  if parent_bid
286
- redis do |r|
287
- r.multi do
288
- r.sadd("BID-#{parent_bid}-dead", jid)
289
- r.expire("BID-#{parent_bid}-dead", BID_EXPIRE_TTL)
290
- end
270
+ redis.multi do
271
+ r.sadd("BID-#{parent_bid}-dead", jid)
272
+ r.expire("BID-#{parent_bid}-dead", BID_EXPIRE_TTL)
291
273
  end
292
274
  end
293
275
 
@@ -413,8 +395,21 @@ module CanvasSync
413
395
  cleanup_redis(bid)
414
396
  end
415
397
 
416
- def redis(*args, &blk)
417
- defined?(::Sidekiq) ? ::Sidekiq.redis(*args, &blk) : nil # TODO
398
+ def redis(&blk)
399
+ return RedisProxy.new unless block_given?
400
+
401
+ if Thread.current[:job_batches_redis]
402
+ yield Thread.current[:job_batches_redis]
403
+ elsif defined?(::Sidekiq)
404
+ ::Sidekiq.redis do |r|
405
+ Thread.current[:job_batches_redis] = r
406
+ yield r
407
+ ensure
408
+ Thread.current[:job_batches_redis] = nil
409
+ end
410
+ else
411
+ # TODO
412
+ end
418
413
  end
419
414
 
420
415
  def logger
@@ -433,6 +428,42 @@ module CanvasSync
433
428
  end
434
429
  end
435
430
  end
431
+
432
+ class RedisProxy
433
+ def multi(*args, &block)
434
+ Batch.redis do |r|
435
+ if block.arity == 1
436
+ r.multi(*args) do
437
+ block.call(r)
438
+ end
439
+ else
440
+ r.multi(*args, &block)
441
+ end
442
+ end
443
+ end
444
+
445
+ def pipelined(*args, &block)
446
+ Batch.redis do |r|
447
+ if block.arity == 1
448
+ r.pipelined(*args) do
449
+ block.call(r)
450
+ end
451
+ else
452
+ r.pipelined(*args, &block)
453
+ end
454
+ end
455
+ end
456
+
457
+ def method_missing(method_name, *arguments, &block)
458
+ Batch.redis do |r|
459
+ r.send(method_name, *arguments, &block)
460
+ end
461
+ end
462
+
463
+ def respond_to_missing?(method_name, include_private = false)
464
+ super || Redis.method_defined?(method_name)
465
+ end
466
+ end
436
467
  end
437
468
 
438
469
  ActiveJob::Base.include BatchAwareJob
@@ -64,18 +64,14 @@ module CanvasSync
64
64
  let_close!
65
65
  end
66
66
  else
67
- redis do |r|
68
- r.hset(redis_key, 'keep_open', true)
69
- end
67
+ redis.hset(redis_key, 'keep_open', true)
70
68
  end
71
69
  end
72
70
 
73
71
  def let_close!
74
- _, active_count = redis do |r|
75
- r.multi do
76
- r.hset(redis_key, 'keep_open', false)
77
- r.hincrby(redis_key, "active_count", 0)
78
- end
72
+ _, active_count = redis.multi do |r|
73
+ r.hset(redis_key, 'keep_open', false)
74
+ r.hincrby(redis_key, "active_count", 0)
79
75
  end
80
76
 
81
77
  if active_count == 0 && pending_count == 0
@@ -95,35 +91,38 @@ module CanvasSync
95
91
  end
96
92
 
97
93
  def active_count
98
- redis do |r|
99
- r.hincrby(redis_key, "active_count", 0)
100
- end
94
+ redis.hincrby(redis_key, "active_count", 0)
101
95
  end
102
96
 
103
97
  def pending_count
104
98
  jobs_key = "#{redis_key}-jobs"
105
99
  order = self.order || 'fifo'
106
- redis do |r|
107
- case order.to_sym
108
- when :fifo, :lifo
109
- r.llen(jobs_key)
110
- when :random
111
- r.scard(jobs_key)
112
- when :priority
113
- r.zcard(jobs_key)
114
- end
100
+ case order.to_sym
101
+ when :fifo, :lifo
102
+ redis.llen(jobs_key)
103
+ when :random
104
+ redis.scard(jobs_key)
105
+ when :priority
106
+ redis.zcard(jobs_key)
115
107
  end
116
108
  end
117
109
 
118
110
  def job_checked_in(status, options)
119
- active_count = redis do |r|
111
+ active_count, pending_count = redis do |r|
120
112
  return unless r.exists?(redis_key)
121
- r.hincrby(redis_key, "active_count", -1)
113
+
114
+ # Make sure this is loaded outside of the pipeline
115
+ self.order
116
+
117
+ redis.multi do
118
+ r.hincrby(redis_key, "active_count", -1)
119
+ self.pending_count
120
+ end
122
121
  end
123
122
 
124
123
  added_count = refill_allotment
125
- if active_count == 0 && added_count == 0
126
- if clean_when_empty && redis {|r| r.hget(redis_key, 'keep_open') } != 'true'
124
+ if active_count == 0 && added_count == 0 && pending_count == 0
125
+ if clean_when_empty && redis.hget(redis_key, 'keep_open') != 'true'
127
126
  cleanup_redis
128
127
  end
129
128
  end
@@ -175,18 +174,16 @@ module CanvasSync
175
174
  job_json = JSON.unparse(ActiveJob::Arguments.serialize([job_desc]))
176
175
  order = self.order
177
176
 
178
- redis do |r|
179
- r.multi do
180
- case order.to_sym
181
- when :fifo, :lifo
182
- r.rpush(jobs_key, job_json)
183
- when :random
184
- r.sadd(jobs_key, job_json)
185
- when :priority
186
- r.zadd(jobs_key, job_desc[:priority] || 0, job_json)
187
- end
188
- r.expire(jobs_key, Batch::BID_EXPIRE_TTL)
177
+ redis.multi do |r|
178
+ case order.to_sym
179
+ when :fifo, :lifo
180
+ r.rpush(jobs_key, job_json)
181
+ when :random
182
+ r.sadd(jobs_key, job_json)
183
+ when :priority
184
+ r.zadd(jobs_key, job_desc[:priority] || 0, job_json)
189
185
  end
186
+ r.expire(jobs_key, Batch::BID_EXPIRE_TTL)
190
187
  end
191
188
  end
192
189
 
@@ -194,18 +191,15 @@ module CanvasSync
194
191
  jobs_key = "#{redis_key}-jobs"
195
192
  order = self.order
196
193
 
197
- job_json = nil
198
- redis do |r|
199
- job_json = case order.to_sym
200
- when :fifo
201
- r.lpop(jobs_key)
202
- when :lifo
203
- r.rpop(jobs_key)
204
- when :random
205
- r.spop(jobs_key)
206
- when :priority
207
- r.zpopmax(jobs_key)
208
- end
194
+ job_json = case order.to_sym
195
+ when :fifo
196
+ redis.lpop(jobs_key)
197
+ when :lifo
198
+ redis.rpop(jobs_key)
199
+ when :random
200
+ redis.spop(jobs_key)
201
+ when :priority
202
+ redis.zpopmax(jobs_key)
209
203
  end
210
204
 
211
205
  return nil unless job_json.present?
@@ -218,13 +212,9 @@ module CanvasSync
218
212
  end
219
213
  delegate :redis, to: :class
220
214
 
221
- protected
222
-
223
215
  def flush_pending_attrs
224
216
  super
225
- redis do |r|
226
- r.zadd("pools", created_at, pid)
227
- end
217
+ redis.zadd("pools", created_at, pid)
228
218
  end
229
219
 
230
220
  private
@@ -9,7 +9,7 @@ module CanvasSync
9
9
  # will be started for each term in that scope. :models should be an array of
10
10
  # models to sync.
11
11
  def perform(options)
12
- CanvasSync.get_canvas_sync_client(batch_context).terms("self").all_pages!.each do |term_params|
12
+ CanvasSync.get_canvas_sync_client(batch_context).terms("self", workflow_state: ['all']).all_pages!.each do |term_params|
13
13
  if account_id = batch_context[:account_id]
14
14
  # These branches are primarily to support Legacy apps
15
15
  if Term.respond_to?(:create_or_update) && Term.method(:create_or_update).arity.abs == 2
@@ -457,12 +457,6 @@ grading_period_groups:
457
457
  grading_period_group_id:
458
458
  database_column_name: canvas_id
459
459
  type: integer
460
- canvas_course_id:
461
- database_column_name: canvas_course_id
462
- type: integer
463
- canvas_account_id:
464
- database_column_name: canvas_account_id
465
- type: integer
466
460
  title:
467
461
  database_column_name: title
468
462
  type: string
@@ -1,3 +1,3 @@
1
1
  module CanvasSync
2
- VERSION = "0.17.23.beta5".freeze
2
+ VERSION = "0.17.23".freeze
3
3
  end
@@ -10,6 +10,4 @@ class GradingPeriodGroup < ApplicationRecord
10
10
  include CanvasSync::Record
11
11
 
12
12
  validates :canvas_id, uniqueness: true, presence: true
13
- belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true
14
- belongs_to :account, primary_key: :canvas_id, foreign_key: :canvas_account_id, optional: true
15
13
  end
@@ -12,6 +12,7 @@ class Term < ApplicationRecord
12
12
 
13
13
  validates :canvas_id, uniqueness: true, presence: true
14
14
  has_many :courses, foreign_key: :canvas_term_id, primary_key: :canvas_id
15
+ belongs_to :grading_period_group, primary_key: :canvas_id, foreign_key: :grading_period_group_id, optional: true
15
16
 
16
17
  api_syncable({
17
18
  canvas_id: :id,
@@ -10,8 +10,6 @@ class CreateGradingPeriodGroups < ActiveRecord::Migration[5.1]
10
10
  def change
11
11
  create_table :grading_period_groups do |t|
12
12
  t.bigint :canvas_id, null: false
13
- t.bigint :canvas_course_id
14
- t.bigint :canvas_account_id
15
13
  t.string :title
16
14
  t.boolean :weighted
17
15
  t.boolean :display_totals_for_all_grading_periods
@@ -22,7 +20,5 @@ class CreateGradingPeriodGroups < ActiveRecord::Migration[5.1]
22
20
  end
23
21
 
24
22
  add_index :grading_period_groups, :canvas_id, unique: true
25
- add_index :grading_period_groups, :canvas_course_id
26
- add_index :grading_period_groups, :canvas_account_id
27
23
  end
28
24
  end
@@ -10,7 +10,7 @@
10
10
  #
11
11
  # It's strongly recommended that you check this file into your version control system.
12
12
 
13
- ActiveRecord::Schema.define(version: 2021_09_07_233331) do
13
+ ActiveRecord::Schema.define(version: 2021_10_01_184920) do
14
14
 
15
15
  # These are extensions that must be enabled in order to support this database
16
16
  enable_extension "plpgsql"
@@ -175,16 +175,12 @@ ActiveRecord::Schema.define(version: 2021_09_07_233331) do
175
175
 
176
176
  create_table "grading_period_groups", force: :cascade do |t|
177
177
  t.bigint "canvas_id", null: false
178
- t.bigint "canvas_course_id"
179
- t.bigint "canvas_account_id"
180
178
  t.string "title"
181
179
  t.boolean "weighted"
182
180
  t.boolean "display_totals_for_all_grading_periods"
183
181
  t.string "workflow_state"
184
182
  t.datetime "created_at", null: false
185
183
  t.datetime "updated_at", null: false
186
- t.index ["canvas_account_id"], name: "index_grading_period_groups_on_canvas_account_id"
187
- t.index ["canvas_course_id"], name: "index_grading_period_groups_on_canvas_course_id"
188
184
  t.index ["canvas_id"], name: "index_grading_period_groups_on_canvas_id", unique: true
189
185
  end
190
186