canvas_sync 0.17.0.beta2 → 0.17.0.beta7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1efae2b1ce34cbd2732fa1077f99968da3cc167e4a95bb05e1da66589775165c
-  data.tar.gz: c24dd070fa499ddfe2a317f6dcf72ba17d99c494dd38bdb1ed2c12e39601b39b
+  metadata.gz: 92aceb45ca42ac3cedcb449ea12ee0c04f437239ccc7375995f56939216ea473
+  data.tar.gz: 37b66818e5ba35d6b0d159ef817cbaf3c8bf66b8194e21c85876f2fdebaa30a0
 SHA512:
-  metadata.gz: 9560928f39ae693290541b14b3d7b381d3259f603881e34c7b4bc5871a9c23a47fae876a5f7334660b8ef17d3e6a03e7ae34bffb8f1c666afad9127ee2934da8
-  data.tar.gz: 2c7874ef5441477e16675f916e450a0f095eabe167e7a16af3c8744fc24c62a90179fa9b2a2ac57a7fff7dda0dab1050c3cbdcc96973dca3d14bb4d7e54b15cb
+  metadata.gz: 22b6826841d14ef7c9bbde34ba0d17484183a8f48247167b08b4332ced3b2b8e5bff7d2d4ea6e6e8246ea720efd79998b46768c973303094839104804c827e26
+  data.tar.gz: da90ec118b97081ce4f54de372e288d0033f76d91e1401ef1bcf70d7a62d589a0e08df18a83030da4e2f0b14bffe51cc33b3b97c198ec7f70f58874b118c2fd2

data/lib/canvas_sync.rb

@@ -161,6 +161,7 @@ module CanvasSync
       ###############################
 
       per_term_chain = JobBatches::ChainBuilder.new(model_job_map[:terms])
+      per_term_chain.params[:term_scope] = term_scope
       current_chain << per_term_chain
       current_chain = per_term_chain
 
@@ -171,7 +172,7 @@ module CanvasSync
       try_add_model_job.call('context_module_items')
 
       current_chain.insert(
-        generate_provisioning_jobs(models, options, job_options: { term_scope: term_scope }, only_split: ['users'])
+        generate_provisioning_jobs(models, options, only_split: ['users'])
       )
 
       ###############################

data/lib/canvas_sync/job_batches/batch.rb

@@ -118,40 +118,29 @@ module CanvasSync
           @initialized = true
         end
 
-        @ready_to_queue = []
+        job_queue = @ready_to_queue = []
+
+        puts "Beginning Batch #{@bidkey}"
 
         begin
           parent = Thread.current[:batch]
           Thread.current[:batch] = self
           yield
         ensure
+          @ready_to_queue = nil
+          append_jobs(job_queue)
           Thread.current[:batch] = parent
         end
 
-        redis do |r|
-          r.multi do
-            if parent_bid
-              r.hincrby("BID-#{parent_bid}", "children", 1)
-              r.hincrby("BID-#{parent_bid}", "total", @ready_to_queue.size)
-              r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
-            end
-
-            r.hincrby(@bidkey, "pending", @ready_to_queue.size)
-            r.hincrby(@bidkey, "total", @ready_to_queue.size)
-            r.expire(@bidkey, BID_EXPIRE_TTL)
-
-            if @ready_to_queue.size > 0
-              r.sadd(@bidkey + "-jids", @ready_to_queue)
-              r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
-            end
-          end
-        end
-
-        @ready_to_queue
+        job_queue
       end
 
       def increment_job_queue(jid)
-        @ready_to_queue << jid
+        if @ready_to_queue
+          @ready_to_queue << jid
+        else
+          append_jobs([jid])
+        end
       end
 
       def invalidate_all
@@ -214,6 +203,27 @@ module CanvasSync
         @pending_attrs = {}
       end
 
+      def append_jobs(jids)
+        redis do |r|
+          r.multi do
+            if parent_bid
+              r.hincrby("BID-#{parent_bid}", "children", 1)
+              r.hincrby("BID-#{parent_bid}", "total", jids.size)
+              r.expire("BID-#{parent_bid}", BID_EXPIRE_TTL)
+            end
+
+            r.hincrby(@bidkey, "pending", jids.size)
+            r.hincrby(@bidkey, "total", jids.size)
+            r.expire(@bidkey, BID_EXPIRE_TTL)
+
+            if jids.size > 0
+              r.sadd(@bidkey + "-jids", jids)
+              r.expire(@bidkey + "-jids", BID_EXPIRE_TTL)
+            end
+          end
+        end
+      end
+
       class << self
         def process_failed_job(bid, jid)
           _, pending, failed, children, complete, parent_bid = redis do |r|
@@ -223,7 +233,7 @@ module CanvasSync
               r.hincrby("BID-#{bid}", "pending", 0)
               r.scard("BID-#{bid}-failed")
               r.hincrby("BID-#{bid}", "children", 0)
-              r.scard("BID-#{bid}-complete")
+              r.scard("BID-#{bid}-batches-complete")
               r.hget("BID-#{bid}", "parent_bid")
 
               r.expire("BID-#{bid}-failed", BID_EXPIRE_TTL)
@@ -253,7 +263,7 @@ module CanvasSync
 
               r.scard("BID-#{bid}-dead")
               r.hincrby("BID-#{bid}", "children", 0)
-              r.scard("BID-#{bid}-complete")
+              r.scard("BID-#{bid}-batches-complete")
               r.hget("BID-#{bid}", "parent_bid")
 
               r.expire("BID-#{bid}-dead", BID_EXPIRE_TTL)
@@ -273,17 +283,18 @@ module CanvasSync
         end
 
         def process_successful_job(bid, jid)
-          failed, pending, children, complete, success, total, parent_bid = redis do |r|
+          _, failed, pending, children, complete, success, total, parent_bid = redis do |r|
             r.multi do
+              r.srem("BID-#{bid}-failed", jid)
+
               r.scard("BID-#{bid}-failed")
               r.hincrby("BID-#{bid}", "pending", -1)
               r.hincrby("BID-#{bid}", "children", 0)
-              r.scard("BID-#{bid}-complete")
-              r.scard("BID-#{bid}-success")
+              r.scard("BID-#{bid}-batches-complete")
+              r.scard("BID-#{bid}-batches-success")
               r.hget("BID-#{bid}", "total")
               r.hget("BID-#{bid}", "parent_bid")
 
-              r.srem("BID-#{bid}-failed", jid)
               r.srem("BID-#{bid}-jids", jid)
               r.expire("BID-#{bid}", BID_EXPIRE_TTL)
             end
@@ -358,18 +369,19 @@ module CanvasSync
 
         def cleanup_redis(bid)
           logger.debug {"Cleaning redis of batch #{bid}"}
-          redis do |r|
-            r.del(
-              "BID-#{bid}",
-              "BID-#{bid}-callbacks-complete",
-              "BID-#{bid}-callbacks-success",
-              "BID-#{bid}-failed",
-
-              "BID-#{bid}-success",
-              "BID-#{bid}-complete",
-              "BID-#{bid}-jids",
-            )
-          end
+          # redis do |r|
+          #   r.del(
+          #     "BID-#{bid}",
+          #     "BID-#{bid}-callbacks-complete",
+          #     "BID-#{bid}-callbacks-success",
+          #     "BID-#{bid}-failed",
+
+          #     "BID-#{bid}-batches-success",
+          #     "BID-#{bid}-batches-complete",
+          #     "BID-#{bid}-batches-failed",
+          #     "BID-#{bid}-jids",
+          #   )
+          # end
         end
 
         def redis(*args, &blk)

data/lib/canvas_sync/job_batches/callback.rb

@@ -87,13 +87,16 @@ module CanvasSync
           def success(bid, status, parent_bid)
             return unless parent_bid
 
-            _, _, success, _, complete, pending, children, failure = Batch.redis do |r|
+            _, _, success, _, _, complete, pending, children, failure = Batch.redis do |r|
               r.multi do
-                r.sadd("BID-#{parent_bid}-success", bid)
-                r.expire("BID-#{parent_bid}-success", Batch::BID_EXPIRE_TTL)
-                r.scard("BID-#{parent_bid}-success")
-                r.sadd("BID-#{parent_bid}-complete", bid)
-                r.scard("BID-#{parent_bid}-complete")
+                r.sadd("BID-#{parent_bid}-batches-success", bid)
+                r.expire("BID-#{parent_bid}-batches-success", Batch::BID_EXPIRE_TTL)
+                r.scard("BID-#{parent_bid}-batches-success")
+
+                r.srem("BID-#{parent_bid}-batches-failed", bid)
+                r.sadd("BID-#{parent_bid}-batches-complete", bid)
+                r.scard("BID-#{parent_bid}-batches-complete")
+
                 r.hincrby("BID-#{parent_bid}", "pending", 0)
                 r.hincrby("BID-#{parent_bid}", "children", 0)
                 r.scard("BID-#{parent_bid}-failed")
@@ -112,7 +115,7 @@ module CanvasSync
               r.multi do
                 r.hincrby("BID-#{bid}", "pending", 0)
                 r.hincrby("BID-#{bid}", "children", 0)
-                r.scard("BID-#{bid}-success")
+                r.scard("BID-#{bid}-batches-success")
               end
             end
 
@@ -127,10 +130,11 @@ module CanvasSync
               # callback may add more jobs to the parent batch
 
               Batch.logger.debug {"Finalize parent complete bid: #{parent_bid}"}
-              _, complete, pending, children, failure = Batch.redis do |r|
+              _, _, complete, pending, children, failure = Batch.redis do |r|
                 r.multi do
-                  r.sadd("BID-#{parent_bid}-complete", bid)
-                  r.scard("BID-#{parent_bid}-complete")
+                  r.sadd("BID-#{parent_bid}-batches-complete", bid)
+                  r.sadd("BID-#{parent_bid}-batches-failed", bid)
+                  r.scard("BID-#{parent_bid}-batches-complete")
                   r.hincrby("BID-#{parent_bid}", "pending", 0)
                   r.hincrby("BID-#{parent_bid}", "children", 0)
                   r.scard("BID-#{parent_bid}-failed")
@@ -141,11 +145,6 @@ module CanvasSync
               end
             end
           end
-
-          def cleanup_redis bid, callback_bid=nil
-            Batch.cleanup_redis bid
-            Batch.cleanup_redis callback_bid if callback_bid
-          end
         end
       end
     end

data/lib/canvas_sync/job_batches/chain_builder.rb

@@ -104,6 +104,16 @@ module CanvasSync
         end
       end
 
+      # Legacy Support
+      def merge_options(job, options)
+        matching_jobs = find_matching_jobs(job)
+
+        matching_jobs.each do |j|
+          j[:options] ||= {}
+          j[:options].deep_merge!(options)
+        end
+      end
+
       private
 
       def find_matching_jobs(search_job, parent_job = self.base_job)
@@ -145,6 +155,13 @@ module CanvasSync
         def enqueue_job(job_def)
           job_class = job_def[:job].constantize
           job_options = job_def[:parameters] || []
+
+          # Legacy Support
+          if job_def[:options]
+            job_options << {} unless job_options[-1].is_a?(Hash)
+            job_options[-1].merge!(job_def[:options])
+          end
+
           if job_class.respond_to? :perform_async
             job_class.perform_async(*job_options)
           else

data/lib/canvas_sync/job_batches/jobs/concurrent_batch_job.rb

@@ -5,6 +5,7 @@ module CanvasSync
     class ConcurrentBatchJob < BaseJob
       def perform(sub_jobs, context: nil)
         Batch.new.tap do |b|
+          b.description = "Concurrent Batch Root"
           b.context = context
           b.jobs do
             sub_jobs.each do |j|

data/lib/canvas_sync/job_batches/jobs/serial_batch_job.rb

@@ -21,6 +21,7 @@ module CanvasSync
           end
         end
 
+        root_batch.description = "Serial Batch Root (#{serial_id})"
         root_batch.allow_context_changes = true
         root_batch.context = context
         root_batch.on(:success, "#{self.class.to_s}.cleanup_redis", serial_batch_id: serial_id)
@@ -61,7 +62,8 @@ module CanvasSync
 
         Batch.new(root_bid).jobs do
           Batch.new.tap do |batch|
-            batch.on(:success, "#{self.to_s}.job_succeeded_callback", serial_batch_id: serial_id)
+          batch.description = "Serial Batch Fiber (#{serial_id})"
+          batch.on(:success, "#{self.to_s}.job_succeeded_callback", serial_batch_id: serial_id)
             batch.jobs do
               ChainBuilder.enqueue_job(next_job)
             end

data/lib/canvas_sync/job_batches/sidekiq.rb

@@ -84,6 +84,8 @@ module CanvasSync
           end
         end
         ::Sidekiq.const_set(:Batch, CanvasSync::JobBatches::Batch)
+        # This alias helps apartment-sidekiq set itself up correctly
+        ::Sidekiq::Batch.const_set(:Server, CanvasSync::JobBatches::Sidekiq::ServerMiddleware)
         ::Sidekiq::Worker.send(:include, JobBatches::Sidekiq::WorkerExtension)
       end
     end

data/lib/canvas_sync/job_batches/status.rb

@@ -40,10 +40,26 @@ module CanvasSync
           'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'complete') }
         end
 
+        def success?
+          'true' == Batch.redis { |r| r.hget("BID-#{bid}", 'success') }
+        end
+
         def child_count
           Batch.redis { |r| r.hget("BID-#{bid}", 'children') }.to_i
         end
 
+        def completed_children_count
+          Batch.redis { |r| r.scard("BID-#{bid}-batches-complete") }.to_i
+        end
+
+        def successful_children_count
+          Batch.redis { |r| r.scard("BID-#{bid}-batches-success") }.to_i
+        end
+
+        def failed_children_count
+          Batch.redis { |r| r.scard("BID-#{bid}-batches-failed") }.to_i
+        end
+
         def data
           {
             bid: bid,
@@ -52,9 +68,13 @@ module CanvasSync
             pending: pending,
             created_at: created_at,
             complete: complete?,
+            success: success?,
             failure_info: failure_info,
             parent_bid: parent_bid,
-            child_count: child_count
+            child_count: child_count,
+            completed_children_count: completed_children_count,
+            successful_children_count: successful_children_count,
+            failed_children_count: failed_children_count,
           }
         end
       end

data/lib/canvas_sync/jobs/begin_sync_chain_job.rb

@@ -15,6 +15,7 @@ module CanvasSync
         JobBatches::Batch.new.tap do |b|
           b.description = "CanvasSync Root Batch"
           b.on(:complete, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
+          b.on(:success, "#{self.class.to_s}.batch_completed", sync_batch_id: sync_batch.id)
           b.context = globals
           b.jobs do
             JobBatches::SerialBatchJob.perform_now(chain_definition)
@@ -25,7 +26,7 @@ module CanvasSync
       def self.batch_completed(status, options)
         sbatch = SyncBatch.find(options['sync_batch_id'])
         sbatch.update!(
-          status: status.failures.positive? ? 'failed' : 'completed',
+          status: status.success? ? 'completed' : 'failed',
           completed_at: DateTime.now,
         )
       end

data/lib/canvas_sync/jobs/sync_provisioning_report_job.rb

@@ -2,9 +2,6 @@ module CanvasSync
   module Jobs
     # ActiveJob class that starts a Canvas provisioning report
     class SyncProvisioningReportJob < CanvasSync::Job
-      # @param options [Hash] If options contains a :term_scope a seperate provisioning report
-      #   will be started for each term in that scope. :models should be an array of
-      #   models to sync.
       def perform(options)
         start_report(report_params(options), options)
       end

data/lib/canvas_sync/jobs/sync_terms_job.rb

@@ -5,7 +5,9 @@ module CanvasSync
       #
       # Terms are pre-synced so that provisioning reports can be scoped to term.
       #
-      # @param options [Hash]
+      # @param options [Hash] If options contains a :term_scope a seperate provisioning report
+      #   will be started for each term in that scope. :models should be an array of
+      #   models to sync.
       def perform(options)
         CanvasSync.get_canvas_sync_client(batch_context).terms("self").all_pages!.each do |term_params|
           if account_id = batch_context[:account_id]

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.17.0.beta2".freeze
+  VERSION = "0.17.0.beta7".freeze
 end

data/spec/canvas_sync/canvas_sync_spec.rb

@@ -36,8 +36,9 @@ RSpec.describe CanvasSync do
             {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
               {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users"], :b=>2}},
               {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
+                :term_scope=>"active",
                 :sub_jobs=>[
-                  {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active", :c=>3}}
+                  {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :c=>3}}
                 ]
               }]}
             ]]}
@@ -54,8 +55,9 @@ RSpec.describe CanvasSync do
               {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
                 {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users"]}},
                 {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
+                  :term_scope=>"active",
                   :sub_jobs=>[
-                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active"}}
+                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"]}}
                   ]
                 }]}
               ]]}
@@ -72,8 +74,9 @@ RSpec.describe CanvasSync do
             :parameters => [[
               {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
                 {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
+                  :term_scope=>nil,
                   :sub_jobs=>[
-                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "courses"], :term_scope=>nil}}
+                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "courses"]}}
                   ]
                 }]}
               ]]}
@@ -91,8 +94,9 @@ RSpec.describe CanvasSync do
               {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
                 {:job=>"CanvasSync::Jobs::SyncRolesJob", :options=>{}},
                 {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
+                  :term_scope=>"active",
                   :sub_jobs=>[
-                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active"}}
+                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"]}}
                   ]
                 }]}
               ]]}
@@ -110,8 +114,9 @@ RSpec.describe CanvasSync do
               {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
                 {:job=>"CanvasSync::Jobs::SyncRolesJob", :options=>{}},
                 {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
+                  :term_scope=>nil,
                   :sub_jobs=>[
-                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>nil}}
+                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"]}}
                   ]
                 }]}
               ]]}
@@ -129,8 +134,9 @@ RSpec.describe CanvasSync do
               {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
                 {:job=>"CanvasSync::Jobs::SyncAdminsJob", :options=>{}},
                 {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
+                  :term_scope=>"active",
                   :sub_jobs=>[
-                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>"active"}}
+                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"]}}
                   ]
                 }]}
               ]]}
@@ -148,8 +154,9 @@ RSpec.describe CanvasSync do
               {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
                 {:job=>"CanvasSync::Jobs::SyncAdminsJob", :options=>{}},
                 {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
+                  :term_scope=>nil,
                   :sub_jobs=>[
-                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :term_scope=>nil}}
+                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"]}}
                   ]
                 }]}
               ]]}
@@ -167,9 +174,10 @@ RSpec.describe CanvasSync do
             :parameters => [[
               {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
                 {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
+                  :term_scope=>nil,
                   :sub_jobs=>[
                     {:job=>"CanvasSync::Jobs::SyncAssignmentsJob", :options=>{}},
-                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"], :term_scope=>nil}}
+                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"]}}
                   ]
                 }]}
               ]]}
@@ -187,9 +195,10 @@ RSpec.describe CanvasSync do
             :parameters => [[
               {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
                 {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
+                  :term_scope=>nil,
                   :sub_jobs=>[
                     {:job=>"CanvasSync::Jobs::SyncSubmissionsJob", :options=>{}},
-                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"], :term_scope=>nil}}
+                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"]}}
                   ]
                 }]}
               ]]}
@@ -207,9 +216,10 @@ RSpec.describe CanvasSync do
             :parameters => [[
               {:job=>"CanvasSync::JobBatches::ConcurrentBatchJob", :parameters=>[[
                 {:job=>"CanvasSync::Jobs::SyncTermsJob", :parameters=>[{
+                  :term_scope=>nil,
                   :sub_jobs=>[
                     {:job=>"CanvasSync::Jobs::SyncAssignmentGroupsJob", :options=>{}},
-                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"], :term_scope=>nil}}
+                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"]}}
                   ]
                 }]}
               ]]}