canvas_sync 0.17.23.beta4 → 0.17.23.beta8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c53a6ea9523ce389c70e26035183573de72ee06c547e380842216b879a6d8bdc
-  data.tar.gz: 166c315f6ba7b0efa337c55a56b8cd418e080f8f98393c4eaed1fd9fdc75a862
+  metadata.gz: 9bed2b1c5be095c602aca19968b0155ff48026a1c5136cb65b5f31e5decbcb40
+  data.tar.gz: ae6112e2694f0118a43619cf3de053d7eee7ff1873e0dface27b0b3558727132
 SHA512:
-  metadata.gz: f7137c46728c087f7e53902099744c4545db67d01da9117f243bbdf9e6ab099c5668a9f9be44752b6d4a4ca4208d1e98ebb2831e777be531416f01cc3befd21c
-  data.tar.gz: 62eecd4a5581297f95ee82c2e81c5c7821565defc9885024153ab17a708c00b5eda22f18e83773decb1884a73d6a7a74453d108c26e50f8f868934cb03a60e54
+  metadata.gz: 6261b328341a102a9ea6c5f91315444604a3556ccbc040fb38122d18db557bd2c5d2cbc7e5b3c74a52d0d48eb0315793232a3cdff0be21964af1c2617d4ae22d
+  data.tar.gz: 458a2c7b422c2a769ec961bd75bd8f43e5dde63c9231bbd4aab6e2a400ec79ee74d9dc12d93cdbd2cc26ef69bb3611972b84956f55cfc2e2a89a4fc22ec2eafb

data/config/initializers/apartment.rb

@@ -5,6 +5,15 @@
 #   * https://github.com/influitive/apartment/issues/508
 
 Rails.application.config.after_initialize do
+  next unless defined?(Apartment)
+
+  # Newer versions of Apartment already solves this issue (and in a better way)
+  begin
+    require('apartment/version')
+    next if Gem::Version.new(Apartment::VERSION) >= Gem::Version.new('2.8.1')
+  rescue LoadError
+  end
+
   begin
     Rails.application.eager_load!
     ActiveRecord::Base.descendants.each do |model|

data/lib/canvas_sync/generators/templates/migrations/create_grading_period_groups.rb

@@ -4,8 +4,6 @@ class CreateGradingPeriodGroups < ActiveRecord::Migration[5.1]
   def change
     create_table :grading_period_groups do |t|
       t.bigint :canvas_id, null: false
-      t.bigint :canvas_course_id
-      t.bigint :canvas_account_id
       t.string :title
       t.boolean :weighted
       t.boolean :display_totals_for_all_grading_periods
@@ -16,7 +14,5 @@ class CreateGradingPeriodGroups < ActiveRecord::Migration[5.1]
     end
 
     add_index :grading_period_groups, :canvas_id, unique: true
-    add_index :grading_period_groups, :canvas_course_id
-    add_index :grading_period_groups, :canvas_account_id
   end
 end

data/lib/canvas_sync/generators/templates/models/grading_period_group.rb

@@ -4,6 +4,4 @@ class GradingPeriodGroup < ApplicationRecord
   include CanvasSync::Record
 
   validates :canvas_id, uniqueness: true, presence: true
-  belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true
-  belongs_to :account, primary_key: :canvas_id, foreign_key: :canvas_account_id, optional: true
 end

data/lib/canvas_sync/generators/templates/models/term.rb

@@ -6,6 +6,7 @@ class Term < ApplicationRecord
 
   validates :canvas_id, uniqueness: true, presence: true
   has_many :courses, foreign_key: :canvas_term_id, primary_key: :canvas_id
+  belongs_to :grading_period_group, primary_key: :canvas_id, foreign_key: :grading_period_group_id, optional: true
 
   api_syncable({
     canvas_id: :id,

data/lib/canvas_sync/job_batches/batch.rb

@@ -76,14 +76,12 @@ module CanvasSync
       def on(event, callback, options = {})
         return unless Callback::VALID_CALLBACKS.include?(event.to_s)
         callback_key = "#{@bidkey}-callbacks-#{event}"
-        redis do |r|
-          r.multi do
-            r.sadd(callback_key, JSON.unparse({
-              callback: callback,
-              opts: options
-            }))
-            r.expire(callback_key, BID_EXPIRE_TTL)
-          end
+        redis.multi do |r|
+          r.sadd(callback_key, JSON.unparse({
+            callback: callback,
+            opts: options
+          }))
+          r.expire(callback_key, BID_EXPIRE_TTL)
         end
       end
 
@@ -93,16 +91,14 @@ module CanvasSync
         if !@existing && !@initialized
           parent_bid = Thread.current[:batch]&.bid
 
-          redis do |r|
-            r.multi do
-              r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
-              r.expire(@bidkey, BID_EXPIRE_TTL)
+          redis.multi do |r|
+            r.hset(@bidkey, "parent_bid", parent_bid.to_s) if parent_bid
+            r.expire(@bidkey, BID_EXPIRE_TTL)
 
-              if parent_bid
-                r.hincrby("BID-#{parent_bid}", "children", 1)
-                r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
-                r.zadd("BID-#{parent_bid}-bids", created_at, bid)
-              end
+            if parent_bid
+              r.hincrby("BID-#{parent_bid}", "children", 1)
+              r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
+              r.zadd("BID-#{parent_bid}-bids", created_at, bid)
             end
           end
 
@@ -131,15 +127,11 @@ module CanvasSync
       end
 
       def invalidate_all
-        redis do |r|
-          r.setex("invalidated-bid-#{bid}", BID_EXPIRE_TTL, 1)
-        end
+        redis.setex("invalidated-bid-#{bid}", BID_EXPIRE_TTL, 1)
       end
 
       def parent_bid
-        redis do |r|
-          r.hget(@bidkey, "parent_bid")
-        end
+        redis.hget(@bidkey, "parent_bid")
       end
 
       def parent
@@ -149,7 +141,7 @@ module CanvasSync
       end
 
       def valid?(batch = self)
-        valid = !redis { |r| r.exists?("invalidated-bid-#{batch.bid}") }
+        valid = !redis.exists?("invalidated-bid-#{batch.bid}")
         batch.parent ? valid && valid?(batch.parent) : valid
       end
 
@@ -162,23 +154,19 @@ module CanvasSync
             let_close!
           end
         else
-          redis do |r|
-            r.hset(@bidkey, 'keep_open', true)
-          end
+          redis.hset(@bidkey, 'keep_open', true)
         end
       end
 
       def let_close!
-        _, failed, pending, children, complete, success = redis do |r|
-          r.multi do
-            r.hset(@bidkey, 'keep_open', false)
-
-            r.scard("BID-#{bid}-failed")
-            r.hincrby("BID-#{bid}", "pending", 0)
-            r.hincrby("BID-#{bid}", "children", 0)
-            r.scard("BID-#{bid}-batches-complete")
-            r.scard("BID-#{bid}-batches-success")
-          end
+        _, failed, pending, children, complete, success = redis.multi do |r|
+          r.hset(@bidkey, 'keep_open', false)
+
+          r.scard("BID-#{bid}-failed")
+          r.hincrby("BID-#{bid}", "pending", 0)
+          r.hincrby("BID-#{bid}", "children", 0)
+          r.scard("BID-#{bid}-batches-complete")
+          r.scard("BID-#{bid}-batches-success")
         end
 
         all_success = pending.to_i.zero? && children == success
@@ -211,18 +199,14 @@ module CanvasSync
 
       def flush_pending_attrs
         super
-        redis do |r|
-          r.zadd("batches", created_at, bid)
-        end
+        redis.zadd("batches", created_at, bid)
       end
 
       private
 
       def assert_batch_is_open
         unless defined?(@closed)
-          redis do |r|
-            @closed = r.hget(@bidkey, 'success') == 'true'
-          end
+          @closed = redis.hget(@bidkey, 'success') == 'true'
         end
         raise "Cannot add jobs to Batch #{} bid - it has already entered the callback-stage" if @closed
       end
@@ -234,7 +218,7 @@ module CanvasSync
         redis do |r|
           tme = Time.now.utc.to_f
           added = r.zadd(@bidkey + "-jids", jids.map{|jid| [tme, jid] }, nx: true)
-          r.multi do
+          r.multi do |r|
             r.hincrby(@bidkey, "pending", added)
             r.hincrby(@bidkey, "job_count", added)
             r.expire(@bidkey, BID_EXPIRE_TTL)
@@ -283,11 +267,9 @@ module CanvasSync
           end
 
           if parent_bid
-            redis do |r|
-              r.multi do
-                r.sadd("BID-#{parent_bid}-dead", jid)
-                r.expire("BID-#{parent_bid}-dead", BID_EXPIRE_TTL)
-              end
+            redis.multi do
+              r.sadd("BID-#{parent_bid}-dead", jid)
+              r.expire("BID-#{parent_bid}-dead", BID_EXPIRE_TTL)
             end
           end
 
@@ -413,8 +395,21 @@ module CanvasSync
           cleanup_redis(bid)
         end
 
-        def redis(*args, &blk)
-          defined?(::Sidekiq) ? ::Sidekiq.redis(*args, &blk) : nil # TODO
+        def redis(&blk)
+          return RedisProxy.new unless block_given?
+
+          if Thread.current[:job_batches_redis]
+            yield Thread.current[:job_batches_redis]
+          elsif defined?(::Sidekiq)
+            ::Sidekiq.redis do |r|
+              Thread.current[:job_batches_redis] = r
+              yield r
+            ensure
+              Thread.current[:job_batches_redis] = nil
+            end
+          else
+            # TODO
+          end
         end
 
         def logger
@@ -433,6 +428,42 @@ module CanvasSync
           end
         end
       end
+
+      class RedisProxy
+        def multi(*args, &block)
+          Batch.redis do |r|
+            if block.arity == 1
+              r.multi(*args) do
+                block.call(r)
+              end
+            else
+              r.multi(*args, &block)
+            end
+          end
+        end
+
+        def pipelined(*args, &block)
+          Batch.redis do |r|
+            if block.arity == 1
+              r.pipelined(*args) do
+                block.call(r)
+              end
+            else
+              r.pipelined(*args, &block)
+            end
+          end
+        end
+
+        def method_missing(method_name, *arguments, &block)
+          Batch.redis do |r|
+            r.send(method_name, *arguments, &block)
+          end
+        end
+
+        def respond_to_missing?(method_name, include_private = false)
+          super || Redis.method_defined?(method_name)
+        end
+      end
     end
 
     ActiveJob::Base.include BatchAwareJob

data/lib/canvas_sync/job_batches/pool.rb

@@ -64,18 +64,14 @@ module CanvasSync
             let_close!
           end
         else
-          redis do |r|
-            r.hset(redis_key, 'keep_open', true)
-          end
+          redis.hset(redis_key, 'keep_open', true)
         end
       end
 
       def let_close!
-        _, active_count = redis do |r|
-          r.multi do
-            r.hset(redis_key, 'keep_open', false)
-            r.hincrby(redis_key, "active_count", 0)
-          end
+        _, active_count = redis.multi do |r|
+          r.hset(redis_key, 'keep_open', false)
+          r.hincrby(redis_key, "active_count", 0)
         end
 
         if active_count == 0 && pending_count == 0
@@ -95,35 +91,38 @@ module CanvasSync
       end
 
       def active_count
-        redis do |r|
-          r.hincrby(redis_key, "active_count", 0)
-        end
+        redis.hincrby(redis_key, "active_count", 0)
       end
 
       def pending_count
         jobs_key = "#{redis_key}-jobs"
         order = self.order || 'fifo'
-        redis do |r|
-          case order.to_sym
-          when :fifo, :lifo
-            r.llen(jobs_key)
-          when :random
-            r.scard(jobs_key)
-          when :priority
-            r.zcard(jobs_key)
-          end
+        case order.to_sym
+        when :fifo, :lifo
+          redis.llen(jobs_key)
+        when :random
+          redis.scard(jobs_key)
+        when :priority
+          redis.zcard(jobs_key)
         end
       end
 
       def job_checked_in(status, options)
-        active_count = redis do |r|
+        active_count, pending_count = redis do |r|
           return unless r.exists?(redis_key)
-          r.hincrby(redis_key, "active_count", -1)
+
+          # Make sure this is loaded outside of the pipeline
+          self.order
+
+          redis.multi do
+            r.hincrby(redis_key, "active_count", -1)
+            self.pending_count
+          end
         end
 
         added_count = refill_allotment
-        if active_count == 0 && added_count == 0
-          if clean_when_empty && redis {|r| r.hget(redis_key, 'keep_open') } != 'true'
+        if active_count == 0 && added_count == 0 && pending_count == 0
+          if clean_when_empty && redis.hget(redis_key, 'keep_open') != 'true'
             cleanup_redis
           end
         end
@@ -175,18 +174,16 @@ module CanvasSync
         job_json = JSON.unparse(ActiveJob::Arguments.serialize([job_desc]))
         order = self.order
 
-        redis do |r|
-          r.multi do
-            case order.to_sym
-            when :fifo, :lifo
-              r.rpush(jobs_key, job_json)
-            when :random
-              r.sadd(jobs_key, job_json)
-            when :priority
-              r.zadd(jobs_key, job_desc[:priority] || 0, job_json)
-            end
-            r.expire(jobs_key, Batch::BID_EXPIRE_TTL)
+        redis.multi do |r|
+          case order.to_sym
+          when :fifo, :lifo
+            r.rpush(jobs_key, job_json)
+          when :random
+            r.sadd(jobs_key, job_json)
+          when :priority
+            r.zadd(jobs_key, job_desc[:priority] || 0, job_json)
           end
+          r.expire(jobs_key, Batch::BID_EXPIRE_TTL)
         end
       end
 
@@ -194,18 +191,15 @@ module CanvasSync
         jobs_key = "#{redis_key}-jobs"
         order = self.order
 
-        job_json = nil
-        redis do |r|
-          job_json = case order.to_sym
-          when :fifo
-            r.lpop(jobs_key)
-          when :lifo
-            r.rpop(jobs_key)
-          when :random
-            r.spop(jobs_key)
-          when :priority
-            r.zpopmax(jobs_key)
-          end
+        job_json = case order.to_sym
+        when :fifo
+          redis.lpop(jobs_key)
+        when :lifo
+          redis.rpop(jobs_key)
+        when :random
+          redis.spop(jobs_key)
+        when :priority
+          redis.zpopmax(jobs_key)
         end
 
         return nil unless job_json.present?
@@ -218,13 +212,9 @@ module CanvasSync
       end
       delegate :redis, to: :class
 
-      protected
-
       def flush_pending_attrs
         super
-        redis do |r|
-          r.zadd("pools", created_at, pid)
-        end
+        redis.zadd("pools", created_at, pid)
       end
 
       private

data/lib/canvas_sync/jobs/sync_terms_job.rb

@@ -9,7 +9,7 @@ module CanvasSync
       #   will be started for each term in that scope. :models should be an array of
       #   models to sync.
       def perform(options)
-        CanvasSync.get_canvas_sync_client(batch_context).terms("self").all_pages!.each do |term_params|
+        CanvasSync.get_canvas_sync_client(batch_context).terms("self", workflow_state: ['all']).all_pages!.each do |term_params|
           if account_id = batch_context[:account_id]
             # These branches are primarily to support Legacy apps
             if Term.respond_to?(:create_or_update) && Term.method(:create_or_update).arity.abs == 2

data/lib/canvas_sync/processors/model_mappings.yml

@@ -457,12 +457,6 @@ grading_period_groups:
     grading_period_group_id:
       database_column_name: canvas_id
       type: integer
-    canvas_course_id:
-      database_column_name: canvas_course_id
-      type: integer
-    canvas_account_id:
-      database_column_name: canvas_account_id
-      type: integer
     title:
       database_column_name: title
       type: string

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.23.beta4".freeze
+  VERSION = "0.17.23.beta8".freeze
 end

data/spec/dummy/app/models/grading_period_group.rb

@@ -10,6 +10,4 @@ class GradingPeriodGroup < ApplicationRecord
   include CanvasSync::Record
 
   validates :canvas_id, uniqueness: true, presence: true
-  belongs_to :course, primary_key: :canvas_id, foreign_key: :canvas_course_id, optional: true
-  belongs_to :account, primary_key: :canvas_id, foreign_key: :canvas_account_id, optional: true
 end

data/spec/dummy/app/models/term.rb

@@ -12,6 +12,7 @@ class Term < ApplicationRecord
 
   validates :canvas_id, uniqueness: true, presence: true
   has_many :courses, foreign_key: :canvas_term_id, primary_key: :canvas_id
+  belongs_to :grading_period_group, primary_key: :canvas_id, foreign_key: :grading_period_group_id, optional: true
 
   api_syncable({
     canvas_id: :id,

data/spec/dummy/db/migrate/{20210907233331_create_grading_period_groups.rb → 20211001184920_create_grading_period_groups.rb}

@@ -10,8 +10,6 @@ class CreateGradingPeriodGroups < ActiveRecord::Migration[5.1]
   def change
     create_table :grading_period_groups do |t|
       t.bigint :canvas_id, null: false
-      t.bigint :canvas_course_id
-      t.bigint :canvas_account_id
       t.string :title
       t.boolean :weighted
       t.boolean :display_totals_for_all_grading_periods
@@ -22,7 +20,5 @@ class CreateGradingPeriodGroups < ActiveRecord::Migration[5.1]
     end
 
     add_index :grading_period_groups, :canvas_id, unique: true
-    add_index :grading_period_groups, :canvas_course_id
-    add_index :grading_period_groups, :canvas_account_id
   end
 end

data/spec/dummy/db/schema.rb

@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_09_07_233331) do
+ActiveRecord::Schema.define(version: 2021_10_01_184920) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -175,16 +175,12 @@ ActiveRecord::Schema.define(version: 2021_09_07_233331) do
 
   create_table "grading_period_groups", force: :cascade do |t|
     t.bigint "canvas_id", null: false
-    t.bigint "canvas_course_id"
-    t.bigint "canvas_account_id"
     t.string "title"
     t.boolean "weighted"
     t.boolean "display_totals_for_all_grading_periods"
     t.string "workflow_state"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index ["canvas_account_id"], name: "index_grading_period_groups_on_canvas_account_id"
-    t.index ["canvas_course_id"], name: "index_grading_period_groups_on_canvas_course_id"
     t.index ["canvas_id"], name: "index_grading_period_groups_on_canvas_id", unique: true
   end