switchman-inst-jobs 3.0.5 → 3.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7507e39f0e9a097dd11cc1389edad14b92d77b959d00deb3cd8610e43b65e042
-  data.tar.gz: 1d6709c61b221f6e1798dfceff1a180507f905a6b398bbd609060fd202c2e8d6
+  metadata.gz: 4efbe05e879c9162df06f945af688c64876b9c56cc870509687160549359fd39
+  data.tar.gz: e32c8a6dfecd0b6c760d07a4d45eeaf6651d6cc7e70e7b6418d82c5a112f70cd
 SHA512:
-  metadata.gz: 5e617b32fbda0fdaa7898bfda6b0d399792be12d03be8d0caa82ac9ad1a4bc9c29143dcca1b79de197eb0be637b61d91e9e06350ae264f02fb949baf4df132b3
-  data.tar.gz: 81b00f0665470553ae32bd88304b4e88a5a8ead5e8fc03cf728161715054a85523fb44a3796252e2a136a2cd58477d05ebb27f1cc13634390e17c3dbbee3561d
+  metadata.gz: 4fe0103d1b4bc305f0c85ba2135366968eb7c33bb6482b6d302dde9b9532db1d3d09fac64d90dad16fe6c6a438236ede5732d16a22d8538053d814010c9d2c87
+  data.tar.gz: feb2e546ee06bc38f964f58e63cfb59fdc25757952762ef8f8faf0521050d264e9f018ea8e6bb0131b277997d63035f68562cf45a7af13886fbfd80f481f5515

data/lib/switchman_inst_jobs/delayed/backend/base.rb

@@ -43,8 +43,7 @@ module SwitchmanInstJobs
           end
 
           def configured_shard_ids
-            (::Delayed::Settings.worker_config.try(:[], 'workers') || []).
-              map { |w| w['shard'] }.compact.uniq
+            ::SwitchmanInstJobs::Delayed::Settings.configured_shard_ids
           end
 
           def processes_locked_locally

data/lib/switchman_inst_jobs/delayed/settings.rb

@@ -0,0 +1,9 @@
+module SwitchmanInstJobs
+  module Delayed
+    module Settings
+      def self.configured_shard_ids
+        (::Delayed::Settings.worker_config.try(:[], 'workers') || []).map { |w| w['shard'] }.compact.uniq
+      end
+    end
+  end
+end

data/lib/switchman_inst_jobs/delayed/worker/health_check.rb

@@ -18,22 +18,23 @@ module SwitchmanInstJobs
             ::Delayed::Settings.worker_health_check_config['service_name'] = original_service_name
           end
 
-          def reschedule_abandoned_jobs(call_super: false)
-            shards = ::Switchman::Shard.delayed_jobs_shards.to_a
-            call_super = shards.first if shards.length == 1
-            unless call_super == false
-              call_super.activate(:delayed_jobs) do
-                return munge_service_name(call_super) { super() }
+          def reschedule_abandoned_jobs
+            shard_ids = ::SwitchmanInstJobs::Delayed::Settings.configured_shard_ids
+            shards = shard_ids.map { |shard_id| ::Delayed::Worker.shard(shard_id) }
+            ::Switchman::Shard.with_each_shard(shards, [:delayed_jobs]) do
+              dj_shard = ::Switchman::Shard.current(:delayed_jobs)
+              dj_shard.activate do
+                munge_service_name(dj_shard) do
+                  # because this rescheduling process is running on every host, we need
+                  # to make sure that it's functioning for each shard the current
+                  # host is programmed to interact with, but ONLY for those shards.
+                  # reading the config lets us iterate over any shards this host should
+                  # work with and lets us pick the correct service name to identify which
+                  # hosts are currently alive and valid via the health checks
+                  super()
+                end
               end
             end
-
-            ::Switchman::Shard.with_each_shard(shards, [:delayed_jobs], exception: :ignore) do
-              shard = ::Switchman::Shard.current(:delayed_jobs)
-              singleton = <<~SINGLETON
-                periodic: Delayed::Worker::HealthCheck.reschedule_abandoned_jobs:#{shard.id}
-              SINGLETON
-              delay(singleton: singleton).reschedule_abandoned_jobs(call_super: shard)
-            end
           end
         end
       end

data/lib/switchman_inst_jobs/jobs_migrator.rb

@@ -39,16 +39,12 @@ module SwitchmanInstJobs
 
         # Do the updates in batches and then just clear redis instead of clearing them one at a time
         target_shards.each do |target_shard, shards|
-          ::Switchman::Shard.where(id: shards).update_all(delayed_jobs_shard_id: target_shard, block_stranded: true)
+          updates = { delayed_jobs_shard_id: target_shard, block_stranded: true }
+          updates[:updated_at] = Time.zone.now if ::Switchman::Shard.column_names.include?('updated_at')
+          ::Switchman::Shard.where(id: shards).update_all(updates)
         end
         clear_shard_cache
 
-        # Wait a little over the 60 second in-process shard cache clearing
-        # threshold to ensure that all new stranded jobs are now being
-        # enqueued with next_in_strand: false
-        Rails.logger.debug('Waiting for caches to clear')
-        sleep(65) unless @skip_cache_wait
-
         ::Switchman::Shard.clear_cache
         # rubocop:disable Style/CombinableLoops
         # We first migrate strands so that we can stop blocking strands before we migrate unstranded jobs
@@ -59,11 +55,33 @@ module SwitchmanInstJobs
         source_shards.each do |s|
           ::Switchman::Shard.lookup(s).activate(:delayed_jobs) { migrate_everything }
         end
+        ensure_unblock_stranded_for(shard_map.map(&:first))
         # rubocop:enable Style/CombinableLoops
       end
 
-      def clear_shard_cache
+      # if :migrate_strands ran on any shards that fell into scenario 1, then
+      # block_stranded never got flipped, so do that now.
+      def ensure_unblock_stranded_for(shards)
+        shards = ::Switchman::Shard.where(id: shards, block_stranded: true).to_a
+        return unless shards.any?
+
+        ::Switchman::Shard.where(id: shards).update_all(block_stranded: false)
+        clear_shard_cache
+
+        # shards is an array of shard objects that is now stale cause block_stranded has been updated.
+        shards.map(&:delayed_jobs_shard).uniq.each do |dj_shard|
+          unblock_strands(dj_shard)
+        end
+      end
+
+      def clear_shard_cache(debug_message = nil)
         ::Switchman.cache.clear
+        Rails.logger.debug("Waiting for caches to clear #{debug_message}")
+        # Wait a little over the 60 second in-process shard cache clearing
+        # threshold to ensure that all new stranded jobs are now being
+        # enqueued with next_in_strand: false
+        # @skip_cache_wait is for spec usage only
+        sleep(65) unless @skip_cache_wait
       end
 
       # This method expects that all relevant shards already have block_stranded: true
@@ -125,7 +143,7 @@ module SwitchmanInstJobs
                     update_all(next_in_strand: false)
                 end
 
-                # 4) is taken care of here, by leaveing next_in_strand alone and
+                # 4) is taken care of here, by leaving next_in_strand alone and
                 # it should execute on the new shard
                 batch_move_jobs(
                   target_shard: target_shard,
@@ -141,35 +159,32 @@ module SwitchmanInstJobs
             updated = ::Switchman::Shard.where(id: source_shard_ids, block_stranded: true).
               update_all(block_stranded: false)
             # If this is being manually re-run for some reason to clean something up, don't wait for nothing to happen
-            unless updated.zero?
-              clear_shard_cache
-              # Wait a little over the 60 second in-process shard cache clearing
-              # threshold to ensure that all new stranded jobs are now being
-              # enqueued with next_in_strand: false
-              Rails.logger.debug("Waiting for caches to clear (#{source_shard.id} -> #{target_shard.id})")
-              # for spec usage only
-              sleep(65) unless @skip_cache_wait
-            end
+            clear_shard_cache("(#{source_shard.id} -> #{target_shard.id})") unless updated.zero?
+
             ::Switchman::Shard.clear_cache
             # At this time, let's unblock all the strands on the target shard that aren't being held by a blocker
             # but actually could have run and we just didn't know it because we didn't know if they had jobs
             # on the source shard
-            target_shard.activate(:delayed_jobs) do
-              loop do
-                # We only want to unlock stranded jobs where they don't belong to a blocked shard (if they *do* belong)
-                # to a blocked shard, they must be part of a concurrent jobs migration from a different source shard to
-                # this target shard, so we shouldn't unlock them yet.  We only ever unlock one job here to keep the
-                # logic cleaner; if the job is n-stranded, after the first one runs, the trigger will unlock larger
-                # batches
-                break if ::Delayed::Job.where(id: ::Delayed::Job.select('DISTINCT ON (strand) id').
-                  where.not(strand: nil).
-                  where.not(shard_id: ::Switchman::Shard.where(block_stranded: true).pluck(:id)).where(
-                    ::Delayed::Job.select(1).from("#{::Delayed::Job.quoted_table_name} dj2").
-                    where("dj2.next_in_strand = true OR dj2.source = 'JobsMigrator::StrandBlocker'").
-                    where('dj2.strand = delayed_jobs.strand').arel.exists.not
-                  ).order(:strand, :strand_order_override, :id)).limit(500).update_all(next_in_strand: true).zero?
-              end
-            end
+            unblock_strands(target_shard)
+          end
+        end
+      end
+
+      def unblock_strands(target_shard)
+        target_shard.activate(:delayed_jobs) do
+          loop do
+            # We only want to unlock stranded jobs where they don't belong to a blocked shard (if they *do* belong)
+            # to a blocked shard, they must be part of a concurrent jobs migration from a different source shard to
+            # this target shard, so we shouldn't unlock them yet.  We only ever unlock one job here to keep the
+            # logic cleaner; if the job is n-stranded, after the first one runs, the trigger will unlock larger
+            # batches
+            break if ::Delayed::Job.where(id: ::Delayed::Job.select('DISTINCT ON (strand) id').
+              where.not(strand: nil).
+              where.not(shard_id: ::Switchman::Shard.where(block_stranded: true).pluck(:id)).where(
+                ::Delayed::Job.select(1).from("#{::Delayed::Job.quoted_table_name} dj2").
+                where("dj2.next_in_strand = true OR dj2.source = 'JobsMigrator::StrandBlocker'").
+                where('dj2.strand = delayed_jobs.strand').arel.exists.not
+              ).order(:strand, :strand_order_override, :id)).limit(500).update_all(next_in_strand: true).zero?
           end
         end
       end

data/lib/switchman_inst_jobs/switchman/shard.rb

@@ -39,7 +39,14 @@ module SwitchmanInstJobs
 
       def unhold_jobs!
         self.jobs_held = false
-        save! if changed?
+        if changed?
+          save!
+          # Wait a little over the 60 second in-process shard cache clearing
+          # threshold to ensure that all new jobs are now being enqueued
+          # unlocked
+          Rails.logger.debug('Waiting for caches to clear')
+          sleep(65)
+        end
         delayed_jobs_shard.activate(:delayed_jobs) do
           ::Delayed::Job.where(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY, shard_id: id).
             in_batches(of: 10_000).

data/lib/switchman_inst_jobs/version.rb

@@ -1,3 +1,3 @@
 module SwitchmanInstJobs
-  VERSION = '3.0.5'.freeze
+  VERSION = '3.1.2'.freeze
 end

data/lib/switchman_inst_jobs.rb

@@ -38,6 +38,7 @@ end
 
 require 'switchman_inst_jobs/active_record/connection_adapters/postgresql_adapter'
 require 'switchman_inst_jobs/active_record/migration'
+require 'switchman_inst_jobs/delayed/settings'
 require 'switchman_inst_jobs/delayed/backend/base'
 require 'switchman_inst_jobs/delayed/message_sending'
 require 'switchman_inst_jobs/delayed/pool'

metadata

@@ -1,35 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: switchman-inst-jobs
 version: !ruby/object:Gem::Version
-  version: 3.0.5
+  version: 3.1.2
 platform: ruby
 authors:
 - Bryan Petty
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-30 00:00:00.000000000 Z
+date: 2021-01-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inst-jobs
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.3
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -260,7 +260,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.4'
-description:
+description: 
 email:
 - bpetty@instructure.com
 executables: []
@@ -303,6 +303,7 @@ files:
 - lib/switchman_inst_jobs/delayed/backend/base.rb
 - lib/switchman_inst_jobs/delayed/message_sending.rb
 - lib/switchman_inst_jobs/delayed/pool.rb
+- lib/switchman_inst_jobs/delayed/settings.rb
 - lib/switchman_inst_jobs/delayed/worker.rb
 - lib/switchman_inst_jobs/delayed/worker/health_check.rb
 - lib/switchman_inst_jobs/engine.rb
@@ -319,7 +320,7 @@ homepage: https://github.com/instructure/switchman-inst-jobs
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -334,8 +335,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key:
+rubygems_version: 3.0.3
+signing_key: 
 specification_version: 4
 summary: Switchman and Instructure Jobs compatibility gem.
 test_files: []