switchman-inst-jobs 4.0.10 → 4.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f11936008b976dcf8421f1e8c232c04f70e9e1883c9aff5f2191d838bca2155
-  data.tar.gz: d43598b5e9998f83564282b7900c696fd815c55cd80088e14f60c14d1a01d850
+  metadata.gz: e4081fb390625cd34169a6585ffaddb12143403e3545988a1ada3ca90d34dc2d
+  data.tar.gz: 2f60cfa5de1529a206115bd2d68758f41f71763afbdaa980095de67dda1da544
 SHA512:
-  metadata.gz: 0b00fb9a88dcec5e8c9d76175cb9de7194f107b98732cd27a46cfa63ecfd310d080055df26008f9301d16e9780e73ca6605e63ab30963d9a0753bd5b746fcf67
-  data.tar.gz: 5954e29022225b3f7dcf8c9f32c37f97cd0edc6db535753da7633f79167c72141fc96f479e9e334106540205eec91f0901344e0f369f98edec65638a21b9b621
+  metadata.gz: 86bb9901b00dd38410092f17e15e1ff9b70b4907426ad32b7f2779a65844d7442e01e8df056d2293c90354eb814e880068854713e9169099e7d772294fc0a1d9
+  data.tar.gz: 9376187ebecfaab79e07696fd44ed762694e5ed06e9b539183f93f0f49e9706fa9eef59d6e9ef5456bccae2be780360d021da600e854eb8f2afcace71ca25bb5

data/lib/switchman_inst_jobs/delayed/backend/base.rb

@@ -8,6 +8,9 @@ module SwitchmanInstJobs
     end
   end
 
+  class JobsBlockedError < RuntimeError
+  end
+
   module Delayed
     module Backend
       module Base

data/lib/switchman_inst_jobs/jobs_migrator.rb

@@ -9,6 +9,16 @@ module SwitchmanInstJobs
         @before_move_callbacks << proc
       end
 
+      def add_validation_callback(proc)
+        @validation_callbacks ||= []
+        @validation_callbacks << proc
+      end
+
+      def clear_callbacks!
+        @before_move_callbacks = []
+        @validation_callbacks = []
+      end
+
       def transaction_on(shards, &block)
         return yield if shards.empty?
 
@@ -31,6 +41,10 @@ module SwitchmanInstJobs
           source_shards << shard.delayed_jobs_shard.id
           target_shard = target_shard.try(:id) || target_shard
           target_shards[target_shard] += [shard.id]
+
+          @validation_callbacks&.each do |proc|
+            proc.call(shard: shard, target_shard: ::Switchman::Shard.find(target_shard))
+          end
         end
 
         # Do the updates in batches and then just clear redis instead of clearing them one at a time
@@ -81,6 +95,17 @@ module SwitchmanInstJobs
         sleep(65) unless @skip_cache_wait
       end
 
+      def acquire_advisory_lock(type, name)
+        @quoted_function_name ||= ::Delayed::Job.connection.quote_table_name('half_md5_as_bigint')
+
+        value = type == :singleton ? "singleton:#{name}" : name
+        ::Delayed::Job.connection.execute(
+          ::Delayed::Job.sanitize_sql_for_conditions(
+            ["SELECT pg_advisory_xact_lock(#{@quoted_function_name}(?))", value]
+          )
+        )
+      end
+
       # This method expects that all relevant shards already have block_stranded: true
       # but otherwise jobs can be running normally
       def run
@@ -167,15 +192,12 @@ module SwitchmanInstJobs
           locked_by: ::Delayed::Backend::Base::ON_HOLD_BLOCKER
         }
 
-        quoted_function_name = ::Delayed::Job.connection.quote_table_name('half_md5_as_bigint')
         strand_advisory_lock_fn = lambda do |value|
-          ::Delayed::Job.connection.execute("SELECT pg_advisory_xact_lock(#{quoted_function_name}('#{value}'))")
+          acquire_advisory_lock(:strand, value)
         end
 
         singleton_advisory_lock_fn = lambda do |value|
-          ::Delayed::Job.connection.execute(
-            "SELECT pg_advisory_xact_lock(#{quoted_function_name}('singleton:#{value}'))"
-          )
+          acquire_advisory_lock(:singleton, value)
         end
 
         handler.call(strand_scope, :strand, {}, strand_advisory_lock_fn)
@@ -203,12 +225,12 @@ module SwitchmanInstJobs
       end
 
       def unblock_strands(target_shard, batch_size: 10_000)
-        block_stranded_ids = ::Switchman::Shard.where(block_stranded: true).pluck(:id)
+        blocked_shard_ids = blocked_shards.pluck(:id)
         query = lambda { |column, scope|
           ::Delayed::Job.
             where(id: ::Delayed::Job.select("DISTINCT ON (#{column}) id").
               where(scope).
-              where.not(shard_id: block_stranded_ids).
+              where.not(shard_id: blocked_shard_ids).
               where(
                 ::Delayed::Job.select(1).from("#{::Delayed::Job.quoted_table_name} dj2").
                 where("dj2.next_in_strand = true OR dj2.source = 'JobsMigrator::StrandBlocker'").
@@ -250,6 +272,72 @@ module SwitchmanInstJobs
         end
       end
 
+      def blocked_shards
+        ::Switchman::Shard.where(block_stranded: true).or(::Switchman::Shard.where(jobs_held: true))
+      end
+
+      def blocked_by_migrator?(job_scope)
+        job_scope.exists?(source: 'JobsMigrator::StrandBlocker') ||
+          blocked_shards.exists?(id: job_scope.distinct.pluck(:shard_id))
+      end
+
+      def blocked_strands
+        ::Delayed::Job.
+          where.not(strand: nil).
+          group(:strand).
+          having('NOT BOOL_OR(next_in_strand)').
+          pluck(:strand)
+      end
+
+      def unblock_strand!(strand, new_parallelism: nil)
+        job_scope = ::Delayed::Job.where(strand: strand)
+        raise JobsBlockedError if blocked_by_migrator?(job_scope)
+
+        ::Delayed::Job.transaction do
+          acquire_advisory_lock(:strand, strand)
+
+          new_parallelism ||= job_scope.pick('MAX(max_concurrent)')
+          if new_parallelism
+            needed_jobs = new_parallelism - job_scope.where(next_in_strand: true).count
+            if needed_jobs.positive?
+              job_scope.where(next_in_strand: false, locked_by: nil,
+                              singleton: nil).order(:strand_order_override, :id).
+                limit(needed_jobs).update_all(next_in_strand: true)
+            else
+              0
+            end
+          end
+        end
+      end
+
+      def blocked_singletons
+        ::Delayed::Job.
+          where(strand: nil).
+          where.not(singleton: nil).
+          group(:singleton).
+          having('NOT BOOL_OR(next_in_strand)').
+          pluck(:singleton)
+      end
+
+      def unblock_singleton!(singleton)
+        job_scope = ::Delayed::Job.where(strand: nil, singleton: singleton)
+        raise JobsBlockedError if blocked_by_migrator?(job_scope)
+
+        ::Delayed::Job.transaction do
+          acquire_advisory_lock(:singleton, singleton)
+
+          id, next_in_strand = job_scope.
+            group(:singleton).
+            pick('MIN(id), BOOL_OR(next_in_strand)')
+
+          if next_in_strand
+            0
+          elsif id
+            ::Delayed::Job.where(id: id).update_all(next_in_strand: true)
+          end
+        end
+      end
+
       private
 
       def create_blocker_job(**kwargs)

data/lib/switchman_inst_jobs/version.rb

@@ -1,3 +1,3 @@
 module SwitchmanInstJobs
-  VERSION = '4.0.10'.freeze
+  VERSION = '4.0.12'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: switchman-inst-jobs
 version: !ruby/object:Gem::Version
-  version: 4.0.10
+  version: 4.0.12
 platform: ruby
 authors:
 - Bryan Petty
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-09-01 00:00:00.000000000 Z
+date: 2022-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inst-jobs