switchman-inst-jobs 2.0.0 → 3.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f1525c9c7e0f08abefab65094c1479bed9c042585b627022ef5f3d97f05a798
-  data.tar.gz: d47f434d7869d9898e04222b232230c5b54b6c9c11dc390b387438d169dcec67
+  metadata.gz: f447f30d1f541b4fe85ad0d867276957b5ed583ddd01e70c509834c0441892bc
+  data.tar.gz: 677a35494a2b54c609017b335019a98510ca3db04e26e61778ad5a8abc0f9377
 SHA512:
-  metadata.gz: 13ea49df7bfd9c7430c6d6261d11510697ce2c91e62e175cb94d12bc03eb0f6174b5153e35ddf1fe8357a5d20e389b13046aae42d9658016f3ecf7174691c45c
-  data.tar.gz: 7c4ab126b6527b164aa5f278dda90755f15a39d9bc0c2d94cbbabac3d7c3f7c493fcef5009f7d35f4d836c756538d563e00ed3cc96657a7e356ee860c726c7f3
+  metadata.gz: 4dead22acfe97a4a5c06ba7ae348916c39e22cef000754c98d7b4e3ecb9a3312fb2fdffaba1e889812bd77efa9ab2d0ecc9ef6bba3d8325fafb18fe99f70278b
+  data.tar.gz: 05513257e9b30f44e2cb10ee5eb497a66906c38b8e525fb2373326ec81d6cb84df413f02283d46faf3a85d8e996f7da66cbf4b59335e23a743329ad1c1ef6f7c

data/db/migrate/20110610213249_optimize_delayed_jobs.rb

@@ -29,7 +29,7 @@ class OptimizeDelayedJobs < ActiveRecord::Migration[4.2]
     add_index :delayed_jobs, %w[strand id], name: 'index_delayed_jobs_on_strand'
 
     # move all failed jobs to the new failed table
-    Delayed::Backend::ActiveRecord::Job.where('failed_at IS NOT NULL').find_each do |job|
+    Delayed::Backend::ActiveRecord::Job.where.not(failed_at: nil).find_each do |job|
       job.fail! unless job.on_hold?
     end
   end

data/lib/switchman_inst_jobs/delayed/backend/base.rb

@@ -12,13 +12,13 @@ module SwitchmanInstJobs
     module Backend
       module Base
         module ClassMethods
-          def enqueue(object, options = {})
+          def enqueue(object, **options)
             ::Switchman::Shard.periodic_clear_shard_cache
             current_shard = ::Switchman::Shard.current
             enqueue_options = options.merge(
               current_shard: current_shard
             )
-            enqueue_job = -> { ::GuardRail.activate(:master) { super(object, enqueue_options) } }
+            enqueue_job = -> { ::GuardRail.activate(:master) { super(object, **enqueue_options) } }
 
             # Another dj shard must be currently manually activated, so just use that
             # In general this will only happen in unusual circumstances like tests
@@ -43,8 +43,8 @@ module SwitchmanInstJobs
           end
 
           def configured_shard_ids
-            (::Delayed::Settings.worker_config.try(:[], 'workers') || [])
-              .map { |w| w['shard'] }.compact.uniq
+            (::Delayed::Settings.worker_config.try(:[], 'workers') || []).
+              map { |w| w['shard'] }.compact.uniq
           end
 
           def processes_locked_locally
@@ -93,7 +93,7 @@ module SwitchmanInstJobs
           # likely a missing shard with a stale cache
           current_shard.send(:clear_cache)
           ::Switchman::Shard.clear_cache
-          raise ShardNotFoundError, shard_id unless ::Switchman::Shard.where(id: shard_id).exists?
+          raise ShardNotFoundError, shard_id unless ::Switchman::Shard.exists?(id: shard_id)
 
           raise
         end

data/lib/switchman_inst_jobs/delayed/message_sending.rb

@@ -1,8 +1,9 @@
 module SwitchmanInstJobs
   module Delayed
     module MessageSending
-      def send_later_enqueue_args(method, _enqueue_args = {}, *args)
-        return send(method, *args) if ::Switchman::DatabaseServer.creating_new_shard
+      def delay(sender: nil, synchronous: false, **enqueue_args)
+        sender ||= __calculate_sender_for_delay
+        synchronous ||= ::Switchman::DatabaseServer.creating_new_shard
 
         super
       end

data/lib/switchman_inst_jobs/delayed/pool.rb

@@ -17,7 +17,7 @@ module SwitchmanInstJobs
           # We purposely don't .compact to remove nils here, since if any
           # workers are on the default jobs shard we want to unlock against
           # that shard too.
-          shard_ids = @config[:workers].map { |c| c[:shard] }.uniq
+          shard_ids = @config[:workers].pluck(:shard).uniq
           shards = shard_ids.map { |shard_id| ::Delayed::Worker.shard(shard_id) }
         end
         ::Switchman::Shard.with_each_shard(shards, [:delayed_jobs]) do

data/lib/switchman_inst_jobs/delayed/worker/health_check.rb

@@ -19,7 +19,7 @@ module SwitchmanInstJobs
           end
 
           def reschedule_abandoned_jobs(call_super: false)
-            shards = ::Switchman::Shard.delayed_jobs_shards
+            shards = ::Switchman::Shard.delayed_jobs_shards.to_a
             call_super = shards.first if shards.length == 1
             unless call_super == false
               call_super.activate(:delayed_jobs) do
@@ -32,9 +32,7 @@ module SwitchmanInstJobs
               singleton = <<~SINGLETON
                 periodic: Delayed::Worker::HealthCheck.reschedule_abandoned_jobs:#{shard.id}
               SINGLETON
-              send_later_enqueue_args(
-                :reschedule_abandoned_jobs, { singleton: singleton }, call_super: shard
-              )
+              delay(singleton: singleton).reschedule_abandoned_jobs(call_super: shard)
             end
           end
         end

data/lib/switchman_inst_jobs/jobs_migrator.rb

@@ -29,26 +29,41 @@ module SwitchmanInstJobs
 
       def migrate_shards(shard_map)
         source_shards = Set[]
+        target_shards = Hash.new([])
         shard_map.each do |(shard, target_shard)|
           shard = ::Switchman::Shard.find(shard) unless shard.is_a?(::Switchman::Shard)
           source_shards << shard.delayed_jobs_shard.id
-          # If target_shard is an int, it won't have an id, but we can just use it as is
-          shard.update(delayed_jobs_shard_id: target_shard.try(:id) || target_shard, block_stranded: true)
+          target_shard = target_shard.try(:id) || target_shard
+          target_shards[target_shard] += [shard.id]
         end
 
+        # Do the updates in batches and then just clear redis instead of clearing them one at a time
+        target_shards.each do |target_shard, shards|
+          ::Switchman::Shard.where(id: shards).update_all(delayed_jobs_shard_id: target_shard, block_stranded: true)
+        end
+        clear_shard_cache
+
         # Wait a little over the 60 second in-process shard cache clearing
         # threshold to ensure that all new stranded jobs are now being
         # enqueued with next_in_strand: false
-        Rails.logger.debug("Waiting for caches to clear (#{source_shard.id} -> #{target_shard.id})")
+        Rails.logger.debug('Waiting for caches to clear')
         sleep(65) unless @skip_cache_wait
 
-        # TODO: 4 has been picked completely out of a hat.  We should make it configurable or something
-        Parallel.each(source_shards, in_processes: 4) do |s|
-          # Ensure the child processes don't share connections with the parent
-          Delayed::Pool.on_fork.call
-          ActiveRecord::Base.clear_all_connections!
-          s.activate(:delayed_jobs) { run }
+        ::Switchman::Shard.clear_cache
+        # rubocop:disable Style/CombinableLoops
+        # We first migrate strands so that we can stop blocking strands before we migrate unstranded jobs
+        source_shards.each do |s|
+          ::Switchman::Shard.lookup(s).activate(:delayed_jobs) { migrate_strands }
+        end
+
+        source_shards.each do |s|
+          ::Switchman::Shard.lookup(s).activate(:delayed_jobs) { migrate_everything }
         end
+        # rubocop:enable Style/CombinableLoops
+      end
+
+      def clear_shard_cache
+        ::Switchman.cache.clear
       end
 
       # This method expects that all relevant shards already have block_stranded: true
@@ -71,7 +86,7 @@ module SwitchmanInstJobs
         #    those (= do nothing since it should already be true)
 
         source_shard = ::Switchman::Shard.current(:delayed_jobs)
-        strand_scope = ::Delayed::Job.shard(source_shard).where('strand IS NOT NULL')
+        strand_scope = ::Delayed::Job.shard(source_shard).where.not(strand: nil)
         shard_map = build_shard_map(strand_scope, source_shard)
         shard_map.each do |(target_shard, source_shard_ids)|
           shard_scope = strand_scope.where(shard_id: source_shard_ids)
@@ -92,10 +107,10 @@ module SwitchmanInstJobs
                 # We lock it to ensure that the jobs worker can't delete it until we are done moving the strand
                 # Since we only unlock it on the new jobs queue *after* deleting from the original
                 # the lock ensures the blocker always gets unlocked
-                first = this_strand_scope.where('locked_by IS NOT NULL').next_in_strand_order.lock.first
+                first = this_strand_scope.where.not(locked_by: nil).next_in_strand_order.lock.first
                 if first
                   first_job = ::Delayed::Job.create!(strand: strand, next_in_strand: false)
-                  first_job.payload_object = ::Delayed::PerformableMethod.new(Kernel, :sleep, [0])
+                  first_job.payload_object = ::Delayed::PerformableMethod.new(Kernel, :sleep, args: [0])
                   first_job.queue = first.queue
                   first_job.tag = 'Kernel.sleep'
                   first_job.source = 'JobsMigrator::StrandBlocker'
@@ -106,8 +121,8 @@ module SwitchmanInstJobs
                   first_job.save!
                   # the rest of 3) is taken care of here
                   # make sure that all the jobs moved over are NOT next in strand
-                  ::Delayed::Job.where(next_in_strand: true, strand: strand, locked_by: nil)
-                    .update_all(next_in_strand: false)
+                  ::Delayed::Job.where(next_in_strand: true, strand: strand, locked_by: nil).
+                    update_all(next_in_strand: false)
                 end
 
                 # 4) is taken care of here, by leaveing next_in_strand alone and
@@ -123,26 +138,37 @@ module SwitchmanInstJobs
               end
             end
 
-            ::Switchman::Shard.find(source_shard_ids).each do |shard|
-              shard.update(block_stranded: false)
+            updated = ::Switchman::Shard.where(id: source_shard_ids, block_stranded: true).
+              update_all(block_stranded: false)
+            # If this is being manually re-run for some reason to clean something up, don't wait for nothing to happen
+            unless updated.zero?
+              clear_shard_cache
+              # Wait a little over the 60 second in-process shard cache clearing
+              # threshold to ensure that all new stranded jobs are now being
+              # enqueued with next_in_strand: false
+              Rails.logger.debug("Waiting for caches to clear (#{source_shard.id} -> #{target_shard.id})")
+              # for spec usage only
+              sleep(65) unless @skip_cache_wait
             end
-            # Wait a little over the 60 second in-process shard cache clearing
-            # threshold to ensure that all new stranded jobs are now being
-            # enqueued with next_in_strand: false
-            Rails.logger.debug("Waiting for caches to clear (#{source_shard.id} -> #{target_shard.id})")
-            # for spec usage only
-            sleep(65) unless @skip_cache_wait
+            ::Switchman::Shard.clear_cache
             # At this time, let's unblock all the strands on the target shard that aren't being held by a blocker
             # but actually could have run and we just didn't know it because we didn't know if they had jobs
             # on the source shard
-            # rubocop:disable Layout/LineLength
-            strands_to_unblock = shard_scope.where.not(source: 'JobsMigrator::StrandBlocker')
-              .distinct
-              .where("NOT EXISTS (SELECT 1 FROM #{::Delayed::Job.quoted_table_name} dj2 WHERE delayed_jobs.strand=dj2.strand AND next_in_strand)")
-              .pluck(:strand)
-            # rubocop:enable Layout/LineLength
-            strands_to_unblock.each do |strand|
-              Delayed::Job.where(strand: strand).next_in_strand_order.first.update_attribute(:next_in_strand, true)
+            target_shard.activate(:delayed_jobs) do
+              loop do
+                # We only want to unlock stranded jobs where they don't belong to a blocked shard (if they *do* belong)
+                # to a blocked shard, they must be part of a concurrent jobs migration from a different source shard to
+                # this target shard, so we shouldn't unlock them yet.  We only ever unlock one job here to keep the
+                # logic cleaner; if the job is n-stranded, after the first one runs, the trigger will unlock larger
+                # batches
+                break if ::Delayed::Job.where(id: ::Delayed::Job.select('DISTINCT ON (strand) id').
+                  where.not(strand: nil).
+                  where.not(shard_id: ::Switchman::Shard.where(block_stranded: true).pluck(:id)).where(
+                    ::Delayed::Job.select(1).from("#{::Delayed::Job.quoted_table_name} dj2").
+                    where("dj2.next_in_strand = true OR dj2.source = 'JobsMigrator::StrandBlocker'").
+                    where('dj2.strand = delayed_jobs.strand').arel.exists.not
+                  ).order(:strand, :strand_order_override, :id)).limit(500).update_all(next_in_strand: true).zero?
+              end
             end
           end
         end
@@ -242,7 +268,7 @@ module SwitchmanInstJobs
 
         connection.execute "COPY #{::Delayed::Job.quoted_table_name} (#{quoted_keys}) FROM STDIN"
         records.map do |record|
-          connection.raw_connection.put_copy_data(keys.map { |k| quote_text(record[k]) }.join("\t") + "\n")
+          connection.raw_connection.put_copy_data("#{keys.map { |k| quote_text(record[k]) }.join("\t")}\n")
         end
         connection.clear_query_cache
         connection.raw_connection.put_copy_end

data/lib/switchman_inst_jobs/switchman/database_server.rb

@@ -11,7 +11,7 @@ module SwitchmanInstJobs
         # shard's delayed_jobs_shard
         if shard&.default?
           # first look for any shard that behaves like a jobs shard
-          dj_shard ||= ::Switchman::Shard.delayed_jobs_shards.first
+          dj_shard ||= ::Switchman::Shard.delayed_jobs_shards.find(&:database_server)
           # we're really truly out of options, use the default shard itself
           dj_shard ||= shard
         end

data/lib/switchman_inst_jobs/switchman/shard.rb

@@ -28,9 +28,9 @@ module SwitchmanInstJobs
         return unless wait
 
         delayed_jobs_shard.activate(:delayed_jobs) do
-          while ::Delayed::Job.where(shard_id: id)
-              .where.not(locked_at: nil)
-              .where.not(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY).exists?
+          while ::Delayed::Job.where(shard_id: id).
+              where.not(locked_at: nil).
+              where.not(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY).exists?
             sleep 10
             lock_jobs_for_hold
           end
@@ -41,9 +41,9 @@ module SwitchmanInstJobs
         self.jobs_held = false
         save! if changed?
         delayed_jobs_shard.activate(:delayed_jobs) do
-          ::Delayed::Job.where(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY, shard_id: id)
-            .in_batches(of: 10_000)
-            .update_all(
+          ::Delayed::Job.where(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY, shard_id: id).
+            in_batches(of: 10_000).
+            update_all(
               locked_by: nil,
               locked_at: nil,
               attempts: 0,
@@ -111,32 +111,27 @@ module SwitchmanInstJobs
         end
 
         def delayed_jobs_shards
-          unless instance_variable_defined?(:@delayed_jobs_shards)
-            # re-entrancy protection
-            @delayed_jobs_shards = begin
-              shard_dj_shards = [] unless ::Switchman::Shard.columns_hash.key?('delayed_jobs_shard_id')
-              shard_dj_shards ||= begin
-                ::Switchman::Shard
-                  .where.not(delayed_jobs_shard_id: nil)
-                  .distinct
-                  .pluck(:delayed_jobs_shard_id)
-                  .map { |id| ::Switchman::Shard.lookup(id) }
-                  .compact
-              end
-              # set it temporarily, to avoid the default shard falling back to itself
-              # if other shards are usable
-              @delayed_jobs_shards = shard_dj_shards.uniq.sort
-
-              db_dj_shards = ::Switchman::DatabaseServer.all.map do |db|
-                next db.shards.to_a if db.config[:delayed_jobs_shard] == 'self'
-
-                db.delayed_jobs_shard
-              end.compact.flatten.uniq # yes, all three
-
-              (db_dj_shards + shard_dj_shards).uniq.sort
-            end
+          return none unless ::Switchman::Shard.columns_hash.key?('delayed_jobs_shard_id')
+
+          scope = ::Switchman::Shard.unscoped.
+            where(id: ::Switchman::Shard.unscoped.distinct.where.not(delayed_jobs_shard_id: nil).
+            select(:delayed_jobs_shard_id))
+          db_jobs_shards = ::Switchman::DatabaseServer.all.map { |db| db.config[:delayed_jobs_shard] }.uniq
+          db_jobs_shards.delete(nil)
+          has_self = db_jobs_shards.delete('self')
+          scope = scope.or(::Switchman::Shard.unscoped.where(id: db_jobs_shards)) unless db_jobs_shards.empty?
+
+          if has_self
+            self_dbs = ::Switchman::DatabaseServer.all.
+              select { |db| db.config[:delayed_jobs_shard] == 'self' }.map(&:id)
+            scope = scope.or(::Switchman::Shard.unscoped.
+              where(id: ::Switchman::Shard.unscoped.where(delayed_jobs_shard_id: nil, database_server_id: self_dbs).
+              select(:id)))
           end
-          @delayed_jobs_shards
+          @jobs_scope_empty = !scope.exists? unless instance_variable_defined?(:@jobs_scope_empty)
+          return [::Switchman::Shard.default] if @jobs_scope_empty
+
+          ::Switchman::Shard.merge(scope)
         end
       end
     end

data/lib/switchman_inst_jobs/timed_cache.rb

@@ -6,7 +6,7 @@ module SwitchmanInstJobs
       @cached_at = Time.zone.now
     end
 
-    def clear(force = false)
+    def clear(force: false)
       if force || @cached_at < @timeout.call
         @block.call
         @cached_at = Time.zone.now

data/lib/switchman_inst_jobs/version.rb

@@ -1,3 +1,3 @@
 module SwitchmanInstJobs
-  VERSION = '2.0.0'.freeze
+  VERSION = '3.0.4'.freeze
 end

metadata

@@ -1,35 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: switchman-inst-jobs
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 3.0.4
 platform: ruby
 authors:
 - Bryan Petty
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-06 00:00:00.000000000 Z
+date: 2020-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inst-jobs
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.16'
-    - - "<"
+        version: '1.0'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.17'
+        version: 1.0.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.16'
-    - - "<"
+        version: '1.0'
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.17'
+        version: 1.0.3
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
@@ -168,14 +168,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12'
+        version: '13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '12'
+        version: '13'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -210,28 +210,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.79.0
+        version: 1.3.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.79.0
+        version: 1.3.1
 - !ruby/object:Gem::Dependency
   name: rubocop-rails
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.4.2
+        version: 2.8.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.4.2
+        version: 2.8.1
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -334,7 +334,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.1.4
 signing_key:
 specification_version: 4
 summary: Switchman and Instructure Jobs compatibility gem.