canvas_sync 0.22.4 → 0.22.6

data/lib/canvas_sync/job_uniqueness/lock_context.rb

@@ -6,8 +6,6 @@ module CanvasSync::JobUniqueness
       context_class.new(data, **kwargs)
     end
 
-    attr_reader :lock_id
-
     # { job_clazz, jid, queue, args?, kwargs?, base_key? }
     def initialize(data, job_instance: nil, config: nil)
       @base_key = data[:base_key]
@@ -16,7 +14,9 @@ module CanvasSync::JobUniqueness
       @config = config || @context_data[:config]
 
       # TODO Consider (somewhere) updating the lock_id to the BID of the wrapping Batch (when applicable)
-      @lock_id ||= data[:lid] || Thread.current[:unique_jobs_previous_context]&.lock_id || job_id
+      @lock_id ||= data[:lid] || Thread.current[:unique_jobs_previous_context]&.lock_id
+      @lock_id_locked = @lock_id.present?
+      @lock_id ||= job_id
     end
 
     # This is primarily for rehydrating in a Batch Callback, so it is unlikely that args and kwargs are needed.
@@ -84,6 +84,18 @@ module CanvasSync::JobUniqueness
       end
     end
 
+    def lock_id
+      @lock_id_locked = true
+      @lock_id
+    end
+
+    def lock_id=(new_id)
+      raise "Lock ID already set" if @lock_id_locked
+      return unless new_id.present?
+      @lock_id = new_id
+      @lock_id_locked = true
+    end
+
     def job_id
       @context_data[:jid]
     end
@@ -122,14 +134,27 @@ module CanvasSync::JobUniqueness
 
         args = @context_data[:args] || []
         kwargs = @context_data[:kwargs] || {}
+        hash = config[:hash]
         if config[:hash].is_a?(Proc)
-          base_key << config[:hash].call(*args, **kwargs)
+          hash = config[:hash].call(*args, **kwargs)
         elsif config[:hash].nil?
-          base_key << OpenSSL::Digest::MD5.hexdigest(JSON.dump([ args, kwargs.sort]))
-        else
-          base_key << config[:hash]
+          hash = [*args, kwargs]
+        end
+
+        hash = ":#{hash}" if hash.is_a?(Symbol)
+
+        if hash && !hash.is_a?(String)
+          hash = Array(hash)
+
+          # Normalize the hash to ensure that the order of any Hash keys don't matter
+          hash = normalize_hash_chunk(hash)
+
+          normalized = ActiveJob::Arguments.serialize(hash)
+          hash = OpenSSL::Digest::MD5.hexdigest(JSON.dump(normalized))
         end
 
+        base_key << hash if hash
+
         base_key.join(":")
       end
     end
@@ -173,5 +198,14 @@ module CanvasSync::JobUniqueness
 
     private
 
+    def normalize_hash_chunk(chunk)
+      if chunk.is_a?(Hash)
+        chunk.map { |k, v| [k, normalize_hash_chunk(v)] }.sort.to_h
+      elsif chunk.is_a?(Array)
+        chunk.map { |c| normalize_hash_chunk(c) }
+      else
+        chunk
+      end
+    end
   end
 end

data/lib/canvas_sync/jobs/term_batches_job.rb

@@ -12,14 +12,17 @@ module CanvasSync
               # Override the delta-syncing date if:
               # 1. the Term hasn't been synced before or
               # 2. the Term underwent a period of not syncing
-              term_last_sync = CanvasSync.redis.get(self.class.last_sync_key(term_id))
               if batch_context[:updated_after]
+                term_last_sync = CanvasSync.redis.get(self.class.last_sync_key(term_id))
                 if !term_last_sync.present? || batch_context[:updated_after] > term_last_sync
                   local_context[:updated_after] = term_last_sync.presence
                 end
               end
 
               JobBatches::ManagedBatchJob.make_batch(jobs, ordered: false, concurrency: true) do |b|
+                # TODO If we do a Chain UI, this will need to checkin somehow to indicate that the chain forked
+                #   Or chain steps just show a summary - eg "Started", "X Jobs Running", "Done" or "X Jobs Running, Y Jobs Done" - and not individual forks
+                #   For a step to be considered done, all previous sibling-level steps must be done and no batches pending
                 b.description = "TermBatchJob(#{term_id}) Root"
                 b.context = local_context
                 b.on(:success, "#{self.class.to_s}.batch_finished") unless options[:mark_synced] == false

data/lib/canvas_sync/version.rb

@@ -1,3 +1,3 @@
 module CanvasSync
-  VERSION = "0.22.4".freeze
+  VERSION = "0.22.6".freeze
 end

data/spec/canvas_sync/canvas_sync_spec.rb

@@ -2,18 +2,6 @@ require 'spec_helper'
 
 RSpec.describe CanvasSync do
   describe '.provisioning_sync' do
-    it 'invokes the first job in the queue and passes on the rest of the job chain' do
-      expected_job_chain = CanvasSync.default_provisioning_report_chain(['courses'], term_scope: nil).normalize![:args][0]
-
-      expect(CanvasSync::Jobs::BeginSyncChainJob).to receive(:perform_later)
-        .with(
-          expected_job_chain,
-          anything
-        )
-
-      CanvasSync.provisioning_sync(['courses'])
-    end
-
     context 'an invalid model is passed in' do
       it 'raises a helpful exception' do
         expect {
@@ -30,22 +18,25 @@ RSpec.describe CanvasSync do
           provisioning: { c: 3 },
           global: { d: 4 },
         })
-        expect(chain.normalize!).to eq({
+        expect(chain.normalize!).to match({
           :job => "CanvasSync::Jobs::BeginSyncChainJob",
+          :chain_link=>String,
           :args => [
             [
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[
                   [
-                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users"], :b=>2}},
+                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["users"], :b=>2}},
                     {
                       :job=>"CanvasSync::Jobs::SyncTermsJob",
+                      :chain_link=>String,
                       :args=>[],
                       :kwargs=>{
                         :term_scope=>"active",
                         :sub_jobs=>[
-                          {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"], :c=>3}},
+                          {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["courses"], :c=>3}},
                         ],
                       },
                     },
@@ -62,20 +53,23 @@ RSpec.describe CanvasSync do
       context 'we are syncing users with a term scope' do
         it 'syncs the users in a separate job that runs first' do
           chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'], term_scope: :active)
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[
                   [
-                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users"]}},
+                    {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["users"]}},
                     {
                       :job=>"CanvasSync::Jobs::SyncTermsJob",
+                      :chain_link=>String,
                       :args=>[],
                       :kwargs=>{
                         :sub_jobs=>[
-                          {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"]}}
+                          {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["courses"]}}
                         ],
                         :term_scope=>"active",
                       }
@@ -93,17 +87,20 @@ RSpec.describe CanvasSync do
       context 'we are syncing users without a term scope' do
         it 'syncs users along with the rest of the provisioning report' do
           chain = CanvasSync.default_provisioning_report_chain(['users', 'courses'])
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[[
                   {
                     :job=>"CanvasSync::Jobs::SyncTermsJob",
+                    :chain_link=>String,
                     :args=>[],
                     :kwargs=>{
-                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "courses"]}}],
+                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["users", "courses"]}}],
                       :term_scope=>nil,
                     }
                   }
@@ -119,18 +116,21 @@ RSpec.describe CanvasSync do
       context 'we are syncing roles with a term scope' do
         it 'syncs the roles in a separate job that runs first' do
           chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'], term_scope: :active)
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[[
-                  {:job=>"CanvasSync::Jobs::SyncRolesJob", :options=>{}},
+                  {:job=>"CanvasSync::Jobs::SyncRolesJob", :chain_link=>String, :options=>{}},
                   {
                     :job=>"CanvasSync::Jobs::SyncTermsJob",
+                    :chain_link=>String,
                     :args=>[],
                     :kwargs=>{
-                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"]}}],
+                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["courses"]}}],
                       :term_scope=>"active",
                     }
                   },
@@ -146,18 +146,21 @@ RSpec.describe CanvasSync do
       context 'we are syncing roles without a term scope' do
         it 'syncs roles separately even with no term scope' do
           chain = CanvasSync.default_provisioning_report_chain(['roles', 'courses'])
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[[
-                  {:job=>"CanvasSync::Jobs::SyncRolesJob", :options=>{}},
+                  {:job=>"CanvasSync::Jobs::SyncRolesJob", :chain_link=>String, :options=>{}},
                   {
                     :job=>"CanvasSync::Jobs::SyncTermsJob",
+                    :chain_link=>String,
                     :args=>[],
                     :kwargs=>{
-                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"]}}],
+                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["courses"]}}],
                       :term_scope=>nil,
                     }
                   }
@@ -173,18 +176,21 @@ RSpec.describe CanvasSync do
       context 'we are syncing admins with a term scope' do
         it 'syncs the admins in a separate job that runs first' do
           chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'], term_scope: :active)
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[[
-                  {:job=>"CanvasSync::Jobs::SyncAdminsJob", :options=>{}},
+                  {:job=>"CanvasSync::Jobs::SyncAdminsJob", :chain_link=>String, :options=>{}},
                   {
                     :job=>"CanvasSync::Jobs::SyncTermsJob",
+                    :chain_link=>String,
                     :args=>[],
                     :kwargs=>{
-                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"]}}],
+                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["courses"]}}],
                       :term_scope=>"active"
                     }
                   }
@@ -200,18 +206,21 @@ RSpec.describe CanvasSync do
       context 'we are syncing admins without a term scope' do
         it 'syncs admins separately even with no term scope' do
           chain = CanvasSync.default_provisioning_report_chain(['admins', 'courses'])
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[[
-                  {:job=>"CanvasSync::Jobs::SyncAdminsJob", :options=>{}},
+                  {:job=>"CanvasSync::Jobs::SyncAdminsJob", :chain_link=>String, :options=>{}},
                   {
                     :job=>"CanvasSync::Jobs::SyncTermsJob",
+                    :chain_link=>String,
                     :args=>[],
                     :kwargs=>{
-                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["courses"]}}],
+                      :sub_jobs=>[{:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["courses"]}}],
                       :term_scope=>nil
                     }
                   }
@@ -228,19 +237,22 @@ RSpec.describe CanvasSync do
         it "appends the SyncAssignmentsJob" do
           chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments assignments])
 
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[[
                   {
                     :job=>"CanvasSync::Jobs::SyncTermsJob",
+                    :chain_link=>String,
                     :args=>[],
                     :kwargs=>{
                       :sub_jobs=>[
-                        {:job=>"CanvasSync::Jobs::SyncAssignmentsJob", :options=>{}},
-                        {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"]}}
+                        {:job=>"CanvasSync::Jobs::SyncAssignmentsJob", :chain_link=>String, :options=>{}},
+                        {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["users", "enrollments"]}}
                       ],
                       :term_scope=>nil
                     }
@@ -258,19 +270,22 @@ RSpec.describe CanvasSync do
         it "appends the SyncSubmissionsJob" do
           chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments submissions])
 
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[[
                   {
                     :job=>"CanvasSync::Jobs::SyncTermsJob",
+                    :chain_link=>String,
                     :args=>[],
                     :kwargs=>{
                       :sub_jobs=>[
-                        {:job=>"CanvasSync::Jobs::SyncSubmissionsJob", :options=>{}},
-                        {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"]}}
+                        {:job=>"CanvasSync::Jobs::SyncSubmissionsJob", :chain_link=>String, :options=>{}},
+                        {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["users", "enrollments"]}}
                       ],
                       :term_scope=>nil
                     }
@@ -288,19 +303,22 @@ RSpec.describe CanvasSync do
         it "appends the SyncAssignmentGroupsJob" do
           chain = CanvasSync.default_provisioning_report_chain(%w[users enrollments assignment_groups])
 
-          expect(chain.normalize!).to eq({
+          expect(chain.normalize!).to match({
             :job => "CanvasSync::Jobs::BeginSyncChainJob",
+            :chain_link=>String,
             :args => [[
               {
                 :job=>"CanvasSync::JobBatches::ConcurrentBatchJob",
+                :chain_link=>String,
                 :args=>[[
                   {
                     :job=>"CanvasSync::Jobs::SyncTermsJob",
+                    :chain_link=>String,
                     :args=>[],
                     :kwargs=>{
                       :sub_jobs=>[
-                        {:job=>"CanvasSync::Jobs::SyncAssignmentGroupsJob", :options=>{}},
-                        {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :options=>{:models=>["users", "enrollments"]}}
+                        {:job=>"CanvasSync::Jobs::SyncAssignmentGroupsJob", :chain_link=>String, :options=>{}},
+                        {:job=>"CanvasSync::Jobs::SyncProvisioningReportJob", :chain_link=>String, :options=>{:models=>["users", "enrollments"]}}
                       ],
                       :term_scope=>nil
                     }