switchman-inst-jobs 4.0.9 → 4.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c6453b4fb5882411028c48e458ec49ee8dd14686b6baa889053bdc44c0f43ecc
-  data.tar.gz: b5883129b65be01ce10acdb373bdd8bcc95c76e8a18eb4326cb2590b71ead518
+  metadata.gz: e4081fb390625cd34169a6585ffaddb12143403e3545988a1ada3ca90d34dc2d
+  data.tar.gz: 2f60cfa5de1529a206115bd2d68758f41f71763afbdaa980095de67dda1da544
 SHA512:
-  metadata.gz: adc1ed9eb64a227ecbe46f98f92e6b357c595249ad681a4dd3b649fade15b5848ebb4cf15feca35b49ca2c312abe31ef3357941f666be11ad554a1f5cc37892c
-  data.tar.gz: f0d8cd82e01d15b698f57417f2b2d88cb0263bbd47cfba167a812158f89a5110e653590b3e125f65c5555e3f580f346bbf5f1de9b0a58cc8f951bd326d348e9e
+  metadata.gz: 86bb9901b00dd38410092f17e15e1ff9b70b4907426ad32b7f2779a65844d7442e01e8df056d2293c90354eb814e880068854713e9169099e7d772294fc0a1d9
+  data.tar.gz: 9376187ebecfaab79e07696fd44ed762694e5ed06e9b539183f93f0f49e9706fa9eef59d6e9ef5456bccae2be780360d021da600e854eb8f2afcace71ca25bb5

data/lib/switchman_inst_jobs/delayed/backend/base.rb

@@ -8,6 +8,9 @@ module SwitchmanInstJobs
     end
   end
 
+  class JobsBlockedError < RuntimeError
+  end
+
   module Delayed
     module Backend
       module Base

data/lib/switchman_inst_jobs/jobs_migrator.rb

@@ -9,6 +9,16 @@ module SwitchmanInstJobs
         @before_move_callbacks << proc
       end
 
+      def add_validation_callback(proc)
+        @validation_callbacks ||= []
+        @validation_callbacks << proc
+      end
+
+      def clear_callbacks!
+        @before_move_callbacks = []
+        @validation_callbacks = []
+      end
+
       def transaction_on(shards, &block)
         return yield if shards.empty?
 
@@ -31,6 +41,10 @@ module SwitchmanInstJobs
           source_shards << shard.delayed_jobs_shard.id
           target_shard = target_shard.try(:id) || target_shard
           target_shards[target_shard] += [shard.id]
+
+          @validation_callbacks&.each do |proc|
+            proc.call(shard: shard, target_shard: ::Switchman::Shard.find(target_shard))
+          end
         end
 
         # Do the updates in batches and then just clear redis instead of clearing them one at a time
@@ -39,7 +53,7 @@ module SwitchmanInstJobs
           updates[:updated_at] = Time.zone.now if ::Switchman::Shard.column_names.include?('updated_at')
           ::Switchman::Shard.where(id: shards).update_all(updates)
         end
-        clear_shard_cache
+        clear_shard_cache(default: ::Switchman::Shard.exists?(id: target_shards.values.flatten, default: true))
 
         ::Switchman::Shard.clear_cache
         # rubocop:disable Style/CombinableLoops
@@ -62,7 +76,7 @@ module SwitchmanInstJobs
         return unless shards.any?
 
         ::Switchman::Shard.where(id: shards).update_all(block_stranded: false)
-        clear_shard_cache
+        clear_shard_cache(default: shards.any?(&:default?))
 
         # shards is an array of shard objects that is now stale cause block_stranded has been updated.
         shards.map(&:delayed_jobs_shard).uniq.each do |dj_shard|
@@ -70,8 +84,9 @@ module SwitchmanInstJobs
         end
       end
 
-      def clear_shard_cache(debug_message = nil)
-        ::Switchman.cache.clear
+      def clear_shard_cache(debug_message = nil, default:)
+        ::Switchman.cache.delete_matched('shard/*')
+        ::Switchman.cache.delete('default_shard') if default
         Rails.logger.debug { "Waiting for caches to clear #{debug_message}" }
         # Wait a little over the 60 second in-process shard cache clearing
         # threshold to ensure that all new stranded jobs are now being
@@ -80,6 +95,17 @@ module SwitchmanInstJobs
         sleep(65) unless @skip_cache_wait
       end
 
+      def acquire_advisory_lock(type, name)
+        @quoted_function_name ||= ::Delayed::Job.connection.quote_table_name('half_md5_as_bigint')
+
+        value = type == :singleton ? "singleton:#{name}" : name
+        ::Delayed::Job.connection.execute(
+          ::Delayed::Job.sanitize_sql_for_conditions(
+            ["SELECT pg_advisory_xact_lock(#{@quoted_function_name}(?))", value]
+          )
+        )
+      end
+
       # This method expects that all relevant shards already have block_stranded: true
       # but otherwise jobs can be running normally
       def run
@@ -166,15 +192,12 @@ module SwitchmanInstJobs
           locked_by: ::Delayed::Backend::Base::ON_HOLD_BLOCKER
         }
 
-        quoted_function_name = ::Delayed::Job.connection.quote_table_name('half_md5_as_bigint')
         strand_advisory_lock_fn = lambda do |value|
-          ::Delayed::Job.connection.execute("SELECT pg_advisory_xact_lock(#{quoted_function_name}('#{value}'))")
+          acquire_advisory_lock(:strand, value)
         end
 
         singleton_advisory_lock_fn = lambda do |value|
-          ::Delayed::Job.connection.execute(
-            "SELECT pg_advisory_xact_lock(#{quoted_function_name}('singleton:#{value}'))"
-          )
+          acquire_advisory_lock(:singleton, value)
         end
 
         handler.call(strand_scope, :strand, {}, strand_advisory_lock_fn)
@@ -186,7 +209,11 @@ module SwitchmanInstJobs
             updated = ::Switchman::Shard.where(id: source_shard_ids, block_stranded: true).
               update_all(block_stranded: false)
             # If this is being manually re-run for some reason to clean something up, don't wait for nothing to happen
-            clear_shard_cache("(#{source_shard.id} -> #{target_shard.id})") unless updated.zero?
+            unless updated.zero?
+              clear_shard_cache("(#{source_shard.id} -> #{target_shard.id})",
+                                default: ::Switchman::Shard.exists?(id: source_shard_ids,
+                                                                    default: true))
+            end
 
             ::Switchman::Shard.clear_cache
             # At this time, let's unblock all the strands on the target shard that aren't being held by a blocker
@@ -198,12 +225,12 @@ module SwitchmanInstJobs
       end
 
       def unblock_strands(target_shard, batch_size: 10_000)
-        block_stranded_ids = ::Switchman::Shard.where(block_stranded: true).pluck(:id)
+        blocked_shard_ids = blocked_shards.pluck(:id)
         query = lambda { |column, scope|
           ::Delayed::Job.
             where(id: ::Delayed::Job.select("DISTINCT ON (#{column}) id").
               where(scope).
-              where.not(shard_id: block_stranded_ids).
+              where.not(shard_id: blocked_shard_ids).
               where(
                 ::Delayed::Job.select(1).from("#{::Delayed::Job.quoted_table_name} dj2").
                 where("dj2.next_in_strand = true OR dj2.source = 'JobsMigrator::StrandBlocker'").
@@ -245,6 +272,72 @@ module SwitchmanInstJobs
         end
       end
 
+      def blocked_shards
+        ::Switchman::Shard.where(block_stranded: true).or(::Switchman::Shard.where(jobs_held: true))
+      end
+
+      def blocked_by_migrator?(job_scope)
+        job_scope.exists?(source: 'JobsMigrator::StrandBlocker') ||
+          blocked_shards.exists?(id: job_scope.distinct.pluck(:shard_id))
+      end
+
+      def blocked_strands
+        ::Delayed::Job.
+          where.not(strand: nil).
+          group(:strand).
+          having('NOT BOOL_OR(next_in_strand)').
+          pluck(:strand)
+      end
+
+      def unblock_strand!(strand, new_parallelism: nil)
+        job_scope = ::Delayed::Job.where(strand: strand)
+        raise JobsBlockedError if blocked_by_migrator?(job_scope)
+
+        ::Delayed::Job.transaction do
+          acquire_advisory_lock(:strand, strand)
+
+          new_parallelism ||= job_scope.pick('MAX(max_concurrent)')
+          if new_parallelism
+            needed_jobs = new_parallelism - job_scope.where(next_in_strand: true).count
+            if needed_jobs.positive?
+              job_scope.where(next_in_strand: false, locked_by: nil,
+                              singleton: nil).order(:strand_order_override, :id).
+                limit(needed_jobs).update_all(next_in_strand: true)
+            else
+              0
+            end
+          end
+        end
+      end
+
+      def blocked_singletons
+        ::Delayed::Job.
+          where(strand: nil).
+          where.not(singleton: nil).
+          group(:singleton).
+          having('NOT BOOL_OR(next_in_strand)').
+          pluck(:singleton)
+      end
+
+      def unblock_singleton!(singleton)
+        job_scope = ::Delayed::Job.where(strand: nil, singleton: singleton)
+        raise JobsBlockedError if blocked_by_migrator?(job_scope)
+
+        ::Delayed::Job.transaction do
+          acquire_advisory_lock(:singleton, singleton)
+
+          id, next_in_strand = job_scope.
+            group(:singleton).
+            pick('MIN(id), BOOL_OR(next_in_strand)')
+
+          if next_in_strand
+            0
+          elsif id
+            ::Delayed::Job.where(id: id).update_all(next_in_strand: true)
+          end
+        end
+      end
+
       private
 
       def create_blocker_job(**kwargs)

data/lib/switchman_inst_jobs/version.rb

@@ -1,3 +1,3 @@
 module SwitchmanInstJobs
-  VERSION = '4.0.9'.freeze
+  VERSION = '4.0.12'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: switchman-inst-jobs
 version: !ruby/object:Gem::Version
-  version: 4.0.9
+  version: 4.0.12
 platform: ruby
 authors:
 - Bryan Petty
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-07-29 00:00:00.000000000 Z
+date: 2022-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inst-jobs