switchman-inst-jobs 3.1.0 → 3.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9466db757372227599ce8e15411f697c1955dc59bc94dba7143fcc0e4e18b47b
-  data.tar.gz: 0d5e44aecbbf1e9110dcb83e305ada21f98ae52c7b75f7ce052482dc648b4f39
+  metadata.gz: 2a489d2a35d787be5492b4a5d710f1cc6560ad5c3d9f9ad965e0860338562d62
+  data.tar.gz: 88d1ff219ce44f9cb50846c631261ed792f4ce78a8ebde00ea5b08b3a172bf2c
 SHA512:
-  metadata.gz: 883bfa88ae5677c9b8929be849fd468d4b2861a2947c3c2bb626db12fe598c773f7e45706f4ff817558104e072d7c64aa5c73b7823c86fcb18361b06a5f5bd98
-  data.tar.gz: caa20756112d7667fbf00ab5fdfbea825f3515bd9593c5135a399b652e351f8722f49a05088db5347df03c29b900a6139d1dff37b975c723d28a8d2e35a198ae
+  metadata.gz: d0c0574bcb3b5e006072f67fcd6c6a4d892e8db919da73cf4edf4d8ac790c1481fd9777a6b6e6950934951efb647445ff756f59ee9ca663a5f5303588fa5a52b
+  data.tar.gz: b2ffe5cc363ab95d407d21a6c76e486dc9e40dc18d52ceabff9639e6408a33f392c33260a0d243720832e9c760ea8f9c55a941b6b0c31ba8fdae1328308bc5ee

data/lib/switchman_inst_jobs/jobs_migrator.rb

@@ -39,16 +39,12 @@ module SwitchmanInstJobs
 
         # Do the updates in batches and then just clear redis instead of clearing them one at a time
         target_shards.each do |target_shard, shards|
-          ::Switchman::Shard.where(id: shards).update_all(delayed_jobs_shard_id: target_shard, block_stranded: true)
+          updates = { delayed_jobs_shard_id: target_shard, block_stranded: true }
+          updates[:updated_at] = Time.zone.now if ::Switchman::Shard.column_names.include?('updated_at')
+          ::Switchman::Shard.where(id: shards).update_all(updates)
         end
         clear_shard_cache
 
-        # Wait a little over the 60 second in-process shard cache clearing
-        # threshold to ensure that all new stranded jobs are now being
-        # enqueued with next_in_strand: false
-        Rails.logger.debug('Waiting for caches to clear')
-        sleep(65) unless @skip_cache_wait
-
         ::Switchman::Shard.clear_cache
         # rubocop:disable Style/CombinableLoops
         # We first migrate strands so that we can stop blocking strands before we migrate unstranded jobs
@@ -59,11 +55,33 @@ module SwitchmanInstJobs
         source_shards.each do |s|
           ::Switchman::Shard.lookup(s).activate(:delayed_jobs) { migrate_everything }
         end
+        ensure_unblock_stranded_for(shard_map.map(&:first))
         # rubocop:enable Style/CombinableLoops
       end
 
-      def clear_shard_cache
+      # if :migrate_strands ran on any shards that fell into scenario 1, then
+      # block_stranded never got flipped, so do that now.
+      def ensure_unblock_stranded_for(shards)
+        shards = ::Switchman::Shard.where(id: shards, block_stranded: true).to_a
+        return unless shards.any?
+
+        ::Switchman::Shard.where(id: shards).update_all(block_stranded: false)
+        clear_shard_cache
+
+        # shards is an array of shard objects that is now stale cause block_stranded has been updated.
+        shards.map(&:delayed_jobs_shard).uniq.each do |dj_shard|
+          unblock_strands(dj_shard)
+        end
+      end
+
+      def clear_shard_cache(debug_message = nil)
         ::Switchman.cache.clear
+        Rails.logger.debug("Waiting for caches to clear #{debug_message}")
+        # Wait a little over the 60 second in-process shard cache clearing
+        # threshold to ensure that all new stranded jobs are now being
+        # enqueued with next_in_strand: false
+        # @skip_cache_wait is for spec usage only
+        sleep(65) unless @skip_cache_wait
       end
 
       # This method expects that all relevant shards already have block_stranded: true
@@ -125,7 +143,7 @@ module SwitchmanInstJobs
                     update_all(next_in_strand: false)
                 end
 
-                # 4) is taken care of here, by leaveing next_in_strand alone and
+                # 4) is taken care of here, by leaving next_in_strand alone and
                 # it should execute on the new shard
                 batch_move_jobs(
                   target_shard: target_shard,
@@ -141,35 +159,32 @@ module SwitchmanInstJobs
             updated = ::Switchman::Shard.where(id: source_shard_ids, block_stranded: true).
               update_all(block_stranded: false)
             # If this is being manually re-run for some reason to clean something up, don't wait for nothing to happen
-            unless updated.zero?
-              clear_shard_cache
-              # Wait a little over the 60 second in-process shard cache clearing
-              # threshold to ensure that all new stranded jobs are now being
-              # enqueued with next_in_strand: false
-              Rails.logger.debug("Waiting for caches to clear (#{source_shard.id} -> #{target_shard.id})")
-              # for spec usage only
-              sleep(65) unless @skip_cache_wait
-            end
+            clear_shard_cache("(#{source_shard.id} -> #{target_shard.id})") unless updated.zero?
+
             ::Switchman::Shard.clear_cache
             # At this time, let's unblock all the strands on the target shard that aren't being held by a blocker
             # but actually could have run and we just didn't know it because we didn't know if they had jobs
             # on the source shard
-            target_shard.activate(:delayed_jobs) do
-              loop do
-                # We only want to unlock stranded jobs where they don't belong to a blocked shard (if they *do* belong)
-                # to a blocked shard, they must be part of a concurrent jobs migration from a different source shard to
-                # this target shard, so we shouldn't unlock them yet.  We only ever unlock one job here to keep the
-                # logic cleaner; if the job is n-stranded, after the first one runs, the trigger will unlock larger
-                # batches
-                break if ::Delayed::Job.where(id: ::Delayed::Job.select('DISTINCT ON (strand) id').
-                  where.not(strand: nil).
-                  where.not(shard_id: ::Switchman::Shard.where(block_stranded: true).pluck(:id)).where(
-                    ::Delayed::Job.select(1).from("#{::Delayed::Job.quoted_table_name} dj2").
-                    where("dj2.next_in_strand = true OR dj2.source = 'JobsMigrator::StrandBlocker'").
-                    where('dj2.strand = delayed_jobs.strand').arel.exists.not
-                  ).order(:strand, :strand_order_override, :id)).limit(500).update_all(next_in_strand: true).zero?
-              end
-            end
+            unblock_strands(target_shard)
+          end
+        end
+      end
+
+      def unblock_strands(target_shard)
+        target_shard.activate(:delayed_jobs) do
+          loop do
+            # We only want to unlock stranded jobs where they don't belong to a blocked shard (if they *do* belong)
+            # to a blocked shard, they must be part of a concurrent jobs migration from a different source shard to
+            # this target shard, so we shouldn't unlock them yet.  We only ever unlock one job here to keep the
+            # logic cleaner; if the job is n-stranded, after the first one runs, the trigger will unlock larger
+            # batches
+            break if ::Delayed::Job.where(id: ::Delayed::Job.select('DISTINCT ON (strand) id').
+              where.not(strand: nil).
+              where.not(shard_id: ::Switchman::Shard.where(block_stranded: true).pluck(:id)).where(
+                ::Delayed::Job.select(1).from("#{::Delayed::Job.quoted_table_name} dj2").
+                where("dj2.next_in_strand = true OR dj2.source = 'JobsMigrator::StrandBlocker'").
+                where('dj2.strand = delayed_jobs.strand').arel.exists.not
+              ).order(:strand, :strand_order_override, :id)).limit(500).update_all(next_in_strand: true).zero?
           end
         end
       end

data/lib/switchman_inst_jobs/version.rb

@@ -1,3 +1,3 @@
 module SwitchmanInstJobs
-  VERSION = '3.1.0'.freeze
+  VERSION = '3.1.1'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: switchman-inst-jobs
 version: !ruby/object:Gem::Version
-  version: 3.1.0
+  version: 3.1.1
 platform: ruby
 authors:
 - Bryan Petty
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-07 00:00:00.000000000 Z
+date: 2021-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inst-jobs
@@ -260,7 +260,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.4'
-description: 
+description:
 email:
 - bpetty@instructure.com
 executables: []
@@ -319,7 +319,7 @@ homepage: https://github.com/instructure/switchman-inst-jobs
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -334,8 +334,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Switchman and Instructure Jobs compatibility gem.
 test_files: []