switchman-inst-jobs 4.0.10 → 4.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f11936008b976dcf8421f1e8c232c04f70e9e1883c9aff5f2191d838bca2155
-  data.tar.gz: d43598b5e9998f83564282b7900c696fd815c55cd80088e14f60c14d1a01d850
+  metadata.gz: 8663d7c19e1900c68ab1595b7306cc2105233b1aa02bb687beee88cdbfa9ada5
+  data.tar.gz: 63f99e858a89ff31c8cda4a1dd35e072e58cda21ebf9d1cdb6a7434ad37f5968
 SHA512:
-  metadata.gz: 0b00fb9a88dcec5e8c9d76175cb9de7194f107b98732cd27a46cfa63ecfd310d080055df26008f9301d16e9780e73ca6605e63ab30963d9a0753bd5b746fcf67
-  data.tar.gz: 5954e29022225b3f7dcf8c9f32c37f97cd0edc6db535753da7633f79167c72141fc96f479e9e334106540205eec91f0901344e0f369f98edec65638a21b9b621
+  metadata.gz: '0322296a92637bb28cc00cfe41446fc39eb189fb4c5e433fb1adf115f1ab80204d91228c96a5902d5f2ece3906805e478ecf166dee37ef9d17b7ce9d4c28c1bd'
+  data.tar.gz: 769694ebfd1343c2449b18b14b8883aaab0cf3819e01446878328461542657d31df576501db5539f572cf5bae8ca5c460354e960e4c27f3b3996a6c3439894c4

data/lib/switchman_inst_jobs/delayed/backend/base.rb

@@ -8,6 +8,9 @@ module SwitchmanInstJobs
     end
   end
 
+  class JobsBlockedError < RuntimeError
+  end
+
   module Delayed
     module Backend
       module Base

data/lib/switchman_inst_jobs/jobs_migrator.rb

@@ -9,6 +9,16 @@ module SwitchmanInstJobs
         @before_move_callbacks << proc
       end
 
+      def add_validation_callback(proc)
+        @validation_callbacks ||= []
+        @validation_callbacks << proc
+      end
+
+      def clear_callbacks!
+        @before_move_callbacks = []
+        @validation_callbacks = []
+      end
+
       def transaction_on(shards, &block)
         return yield if shards.empty?
 
@@ -31,6 +41,10 @@ module SwitchmanInstJobs
           source_shards << shard.delayed_jobs_shard.id
           target_shard = target_shard.try(:id) || target_shard
           target_shards[target_shard] += [shard.id]
+
+          @validation_callbacks&.each do |proc|
+            proc.call(shard: shard, target_shard: ::Switchman::Shard.find(target_shard))
+          end
         end
 
         # Do the updates in batches and then just clear redis instead of clearing them one at a time
@@ -81,6 +95,17 @@ module SwitchmanInstJobs
         sleep(65) unless @skip_cache_wait
       end
 
+      def acquire_advisory_lock(type, name)
+        @quoted_function_name ||= ::Delayed::Job.connection.quote_table_name('half_md5_as_bigint')
+
+        value = type == :singleton ? "singleton:#{name}" : name
+        ::Delayed::Job.connection.execute(
+          ::Delayed::Job.sanitize_sql_for_conditions(
+            ["SELECT pg_advisory_xact_lock(#{@quoted_function_name}(?))", value]
+          )
+        )
+      end
+
       # This method expects that all relevant shards already have block_stranded: true
       # but otherwise jobs can be running normally
       def run
@@ -167,15 +192,12 @@ module SwitchmanInstJobs
           locked_by: ::Delayed::Backend::Base::ON_HOLD_BLOCKER
         }
 
-        quoted_function_name = ::Delayed::Job.connection.quote_table_name('half_md5_as_bigint')
         strand_advisory_lock_fn = lambda do |value|
-          ::Delayed::Job.connection.execute("SELECT pg_advisory_xact_lock(#{quoted_function_name}('#{value}'))")
+          acquire_advisory_lock(:strand, value)
         end
 
         singleton_advisory_lock_fn = lambda do |value|
-          ::Delayed::Job.connection.execute(
-            "SELECT pg_advisory_xact_lock(#{quoted_function_name}('singleton:#{value}'))"
-          )
+          acquire_advisory_lock(:singleton, value)
         end
 
         handler.call(strand_scope, :strand, {}, strand_advisory_lock_fn)
@@ -203,12 +225,12 @@ module SwitchmanInstJobs
       end
 
       def unblock_strands(target_shard, batch_size: 10_000)
-        block_stranded_ids = ::Switchman::Shard.where(block_stranded: true).pluck(:id)
+        blocked_shard_ids = blocked_shards.pluck(:id)
         query = lambda { |column, scope|
           ::Delayed::Job.
             where(id: ::Delayed::Job.select("DISTINCT ON (#{column}) id").
               where(scope).
-              where.not(shard_id: block_stranded_ids).
+              where.not(shard_id: blocked_shard_ids).
               where(
                 ::Delayed::Job.select(1).from("#{::Delayed::Job.quoted_table_name} dj2").
                 where("dj2.next_in_strand = true OR dj2.source = 'JobsMigrator::StrandBlocker'").
@@ -250,6 +272,76 @@ module SwitchmanInstJobs
         end
       end
 
+      def blocked_shards
+        ::Switchman::Shard.where(block_stranded: true).or(::Switchman::Shard.where(jobs_held: true))
+      end
+
+      def blocked_by_migrator?(job_scope)
+        job_scope.exists?(source: 'JobsMigrator::StrandBlocker') ||
+          blocked_shards.exists?(id: job_scope.distinct.pluck(:shard_id))
+      end
+
+      def blocked_strands
+        ::Delayed::Job.
+          where.not(strand: nil).
+          group(:strand).
+          having('NOT BOOL_OR(next_in_strand)')
+      end
+
+      def unblock_strand!(strand, new_parallelism: nil)
+        job_scope = ::Delayed::Job.where(strand: strand)
+        raise JobsBlockedError if blocked_by_migrator?(job_scope)
+
+        ::Delayed::Job.transaction do
+          acquire_advisory_lock(:strand, strand)
+
+          new_parallelism ||= job_scope.pick('MAX(max_concurrent)')
+          if new_parallelism
+            needed_jobs = new_parallelism - job_scope.where(next_in_strand: true).count
+            if needed_jobs.positive?
+              job_scope.where(next_in_strand: false, locked_by: nil,
+                              singleton: nil).order(:strand_order_override, :id).
+                limit(needed_jobs).update_all(next_in_strand: true)
+            else
+              0
+            end
+          end
+        end
+      end
+
+      def blocked_singletons
+        ::Delayed::Job.
+          where(strand: nil).
+          where.not(singleton: nil).
+          group(:singleton).
+          having('NOT BOOL_OR(next_in_strand)')
+      end
+
+      def unblock_singleton!(singleton)
+        job_scope = ::Delayed::Job.where(strand: nil, singleton: singleton)
+        raise JobsBlockedError if blocked_by_migrator?(job_scope)
+
+        ::Delayed::Job.transaction do
+          acquire_advisory_lock(:singleton, singleton)
+
+          id, next_in_strand = job_scope.
+            group(:singleton).
+            pick('MIN(id), BOOL_OR(next_in_strand)')
+
+          if next_in_strand
+            0
+          elsif id
+            ::Delayed::Job.where(id: id).update_all(next_in_strand: true)
+          end
+        end
+      end
+
+      def blocked_job_count
+        ::Delayed::Job.from(blocked_strands.select('count(id) AS ssize')).sum('ssize').to_i +
+          ::Delayed::Job.from(blocked_singletons.select('count(id) AS ssize')).sum('ssize').to_i +
+          ::Delayed::Job.where(strand: nil, singleton: nil, next_in_strand: false).count
+      end
+
       private
 
       def create_blocker_job(**kwargs)

data/lib/switchman_inst_jobs/version.rb

@@ -1,3 +1,3 @@
 module SwitchmanInstJobs
-  VERSION = '4.0.10'.freeze
+  VERSION = '4.0.13'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: switchman-inst-jobs
 version: !ruby/object:Gem::Version
-  version: 4.0.10
+  version: 4.0.13
 platform: ruby
 authors:
 - Bryan Petty
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-09-01 00:00:00.000000000 Z
+date: 2022-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inst-jobs