switchman-inst-jobs 1.4.3 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6ea858af15b508930d58fd21c5c097c4cedc0f244ef12f7d529120d79483322d
-  data.tar.gz: e431ab3d85859d0dc3acc5e95fbb820bc272c51bc4124615f0bc314c7d6f311a
+  metadata.gz: d279a1bceff0465d419fb835766fda41b6f682ba91ee8b76241238c632d08813
+  data.tar.gz: fb7c5eb1b6a9ab934e951021d489cb1292e4f944abce9163b48f2b2ef16bf976
 SHA512:
-  metadata.gz: 263dc8ef6c816010d058a8033487a3ed89d6978accf1f16f73ebbc311ece6266f57333c80cf9a9f27d738359638e317e3179a2181d6940efbf8c522a02ba6a67
-  data.tar.gz: a7078f7f69c97e18cabbaa0ab3c4e1ef9df08472b29f2fe188cc83353b0759333e806a72e72bccdd6a635e642f75c5008092595be0ad3b6bbda114e27ce8723a
+  metadata.gz: 20c649d58b0975d99e41b13c5fd39eeea73b59763f8075d36301cbdb88b1dd866e4fdaec8c8fa4a656ba9a0ceb166cd099c6b7a80727ce4ed709fbd0da3c0431
+  data.tar.gz: 39a3d57f786c8e02e922e5eaead0a415e3f1337fa4907490f9bc16ab00f153a363db599d5422a52a048b5621ee18e07e130912cbb5a4d07643712f15a3fabac1

data/db/migrate/20200818130101_add_on_hold_to_switchman_shards.rb

@@ -0,0 +1,5 @@
+class AddOnHoldToSwitchmanShards < ActiveRecord::Migration[5.2]
+  def change
+    add_column :switchman_shards, :jobs_held, :bool, default: false
+  end
+end

data/db/migrate/20200822014259_add_block_stranded_to_switchman_shards.rb

@@ -0,0 +1,5 @@
+class AddBlockStrandedToSwitchmanShards < ActiveRecord::Migration[5.2]
+  def change
+    add_column :switchman_shards, :block_stranded, :bool, default: false
+  end
+end

data/lib/switchman_inst_jobs.rb

@@ -44,9 +44,11 @@ require 'switchman_inst_jobs/delayed/pool'
 require 'switchman_inst_jobs/delayed/worker'
 require 'switchman_inst_jobs/delayed/worker/health_check'
 require 'switchman_inst_jobs/engine'
+require 'switchman_inst_jobs/jobs_migrator'
 require 'switchman_inst_jobs/new_relic'
 require 'switchman_inst_jobs/shackles'
 require 'switchman_inst_jobs/switchman/database_server'
 require 'switchman_inst_jobs/switchman/default_shard'
 require 'switchman_inst_jobs/switchman/shard'
+require 'switchman_inst_jobs/timed_cache'
 require 'switchman_inst_jobs/yaml_extensions'

data/lib/switchman_inst_jobs/delayed/backend/base.rb

@@ -13,16 +13,32 @@ module SwitchmanInstJobs
       module Base
         module ClassMethods
           def enqueue(object, options = {})
+            ::Switchman::Shard.periodic_clear_shard_cache
+            current_shard = ::Switchman::Shard.current
             enqueue_options = options.merge(
-              current_shard: ::Switchman::Shard.current
+              current_shard: current_shard
             )
+            enqueue_job = -> { ::Shackles.activate(:master) { super(object, enqueue_options) } }
 
-            if ::ActiveRecord::Migration.open_migrations.positive?
-              ::Switchman::Shard.current.delayed_jobs_shard.activate(:delayed_jobs) do
-                super(object, enqueue_options)
-              end
+            # Another dj shard must be currently manually activated, so just use that
+            # In general this will only happen in unusual circumstances like tests
+            # also if migrations are running, always use the current shard's job shard
+            if ::ActiveRecord::Migration.open_migrations.zero? &&
+               current_shard.delayed_jobs_shard != ::Switchman::Shard.current(:delayed_jobs)
+              enqueue_job.call
             else
-              ::Shackles.activate(:master) { super(object, enqueue_options) }
+              ::Switchman::Shard.default.activate do
+                current_shard = ::Switchman::Shard.lookup(current_shard.id)
+                current_job_shard = current_shard.delayed_jobs_shard
+
+                if (options[:singleton] || options[:strand]) && current_shard.block_stranded
+                  enqueue_options[:next_in_strand] = false
+                end
+
+                current_job_shard.activate(:delayed_jobs) do
+                  enqueue_job.call
+                end
+              end
             end
           end
 
@@ -54,8 +70,15 @@ module SwitchmanInstJobs
         end
 
         def current_shard=(shard)
+          @current_shard = nil
           self.shard_id = shard.id
           self.shard_id = nil if shard.is_a?(::Switchman::DefaultShard)
+          # If jobs are held for a shard, enqueue new ones as held as well
+          return unless shard.jobs_held
+
+          self.locked_by = ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY
+          self.locked_at = ::Delayed::Job.db_time_now
+          self.attempts = ::Delayed::Backend::Base::ON_HOLD_COUNT
         end
 
         def invoke_job

data/lib/switchman_inst_jobs/engine.rb

@@ -16,6 +16,22 @@ module SwitchmanInstJobs
           block.call(worker, config)
         end
       end
+
+      # Ensure jobs get unblocked on the new shard if they exist
+      ::Delayed::Worker.lifecycle.after(:perform) do |_worker, job|
+        if job.strand
+          ::Switchman::Shard.clear_cache
+          ::Switchman::Shard.default.activate do
+            current_job_shard = ::Switchman::Shard.lookup(job.shard_id).delayed_jobs_shard
+            if current_job_shard != ::Switchman::Shard.current(:delayed_jobs)
+              current_job_shard.activate(:delayed_jobs) do
+                j = ::Delayed::Job.where(strand: job.strand).next_in_strand_order.first
+                j.update_column(:next_in_strand, true) if j && !j.next_in_strand
+              end
+            end
+          end
+        end
+      end
     end
 
     initializer 'sharding.shackles', after: 'switchman.extend_shackles' do

data/lib/switchman_inst_jobs/jobs_migrator.rb

@@ -0,0 +1,271 @@
+# Just disabling all the rubocop metrics for this file for now,
+# as it is a direct port-in of existing code
+
+# rubocop:disable Metrics/BlockLength, Metrics/MethodLength, Metrics/AbcSize, Metrics/ClassLength
+require 'set'
+require 'parallel'
+
+module SwitchmanInstJobs
+  class JobsMigrator
+    class << self
+      def add_before_move_callback(proc)
+        @before_move_callbacks ||= []
+        @before_move_callbacks << proc
+      end
+
+      def transaction_on(shards, &block)
+        return yield if shards.empty?
+
+        shard = shards.pop
+        current_shard = ::Switchman::Shard.current(:delayed_jobs)
+        shard.activate(:delayed_jobs) do
+          ::Delayed::Job.transaction do
+            current_shard.activate(:delayed_jobs) do
+              transaction_on(shards, &block)
+            end
+          end
+        end
+      end
+
+      def migrate_shards(shard_map)
+        source_shards = Set[]
+        shard_map.each do |(shard, target_shard)|
+          shard = ::Switchman::Shard.find(shard) unless shard.is_a?(::Switchman::Shard)
+          source_shards << shard.delayed_jobs_shard.id
+          # If target_shard is an int, it won't have an id, but we can just use it as is
+          shard.update(delayed_jobs_shard_id: target_shard.try(:id) || target_shard, block_stranded: true)
+        end
+
+        # Wait a little over the 60 second in-process shard cache clearing
+        # threshold to ensure that all new stranded jobs are now being
+        # enqueued with next_in_strand: false
+        Rails.logger.debug("Waiting for caches to clear (#{source_shard.id} -> #{target_shard.id})")
+        sleep(65) unless @skip_cache_wait
+
+        # TODO: 4 has been picked completely out of a hat.  We should make it configurable or something
+        Parallel.each(source_shards, in_processes: 4) do |s|
+          # Ensure the child processes don't share connections with the parent
+          Delayed::Pool.on_fork.call
+          ActiveRecord::Base.clear_all_connections!
+          s.activate(:delayed_jobs) { run }
+        end
+      end
+
+      # This method expects that all relevant shards already have block_stranded: true
+      # but otherwise jobs can be running normally
+      def run
+        # Ensure this is never run with a dirty in-memory shard cache
+        ::Switchman::Shard.clear_cache
+        migrate_strands
+        migrate_everything
+      end
+
+      def migrate_strands
+        # there are 4 scenarios to deal with here
+        # 1) no running job, no jobs moved: do nothing
+        # 2) running job, no jobs moved; create blocker with next_in_strand=false
+        #    to prevent new jobs from immediately executing
+        # 3) running job, jobs moved; set next_in_strand=false on the first of
+        #    those (= do nothing since it should already be false)
+        # 4) no running job, jobs moved: set next_in_strand=true on the first of
+        #    those (= do nothing since it should already be true)
+
+        source_shard = ::Switchman::Shard.current(:delayed_jobs)
+        strand_scope = ::Delayed::Job.shard(source_shard).where('strand IS NOT NULL')
+        shard_map = build_shard_map(strand_scope, source_shard)
+        shard_map.each do |(target_shard, source_shard_ids)|
+          shard_scope = strand_scope.where(shard_id: source_shard_ids)
+
+          # 1) is taken care of because it should not show up here in strands
+          strands = shard_scope.distinct.order(:strand).pluck(:strand)
+
+          target_shard.activate(:delayed_jobs) do
+            strands.each do |strand|
+              transaction_on([source_shard, target_shard]) do
+                this_strand_scope = shard_scope.where(strand: strand)
+                # we want to copy all the jobs except the one that is still running.
+                jobs_scope = this_strand_scope.where(locked_by: nil)
+
+                # 2) and part of 3) are taken care of here by creating a blocker
+                # job with next_in_strand = false. as soon as the current
+                # running job is finished it should set next_in_strand
+                # We lock it to ensure that the jobs worker can't delete it until we are done moving the strand
+                # Since we only unlock it on the new jobs queue *after* deleting from the original
+                # the lock ensures the blocker always gets unlocked
+                first = this_strand_scope.where('locked_by IS NOT NULL').next_in_strand_order.lock.first
+                if first
+                  first_job = ::Delayed::Job.create!(strand: strand, next_in_strand: false)
+                  first_job.payload_object = ::Delayed::PerformableMethod.new(Kernel, :sleep, [0])
+                  first_job.queue = first.queue
+                  first_job.tag = 'Kernel.sleep'
+                  first_job.source = 'JobsMigrator::StrandBlocker'
+                  first_job.max_attempts = 1
+                  # If we ever have jobs left over from 9999 jobs moves of a single shard,
+                  # something has gone terribly wrong
+                  first_job.strand_order_override = -9999
+                  first_job.save!
+                  # the rest of 3) is taken care of here
+                  # make sure that all the jobs moved over are NOT next in strand
+                  ::Delayed::Job.where(next_in_strand: true, strand: strand, locked_by: nil)
+                    .update_all(next_in_strand: false)
+                end
+
+                # 4) is taken care of here, by leaveing next_in_strand alone and
+                # it should execute on the new shard
+                batch_move_jobs(
+                  target_shard: target_shard,
+                  source_shard: source_shard,
+                  scope: jobs_scope
+                ) do |job, new_job|
+                  # This ensures jobs enqueued on the old jobs shard run before jobs on the new jobs queue
+                  new_job.strand_order_override = job.strand_order_override - 1
+                end
+              end
+            end
+
+            ::Switchman::Shard.find(source_shard_ids).each do |shard|
+              shard.update(block_stranded: false)
+            end
+            # Wait a little over the 60 second in-process shard cache clearing
+            # threshold to ensure that all new stranded jobs are now being
+            # enqueued with next_in_strand: false
+            Rails.logger.debug("Waiting for caches to clear (#{source_shard.id} -> #{target_shard.id})")
+            # for spec usage only
+            sleep(65) unless @skip_cache_wait
+            # At this time, let's unblock all the strands on the target shard that aren't being held by a blocker
+            # but actually could have run and we just didn't know it because we didn't know if they had jobs
+            # on the source shard
+            # rubocop:disable Layout/LineLength
+            strands_to_unblock = shard_scope.where.not(source: 'JobsMigrator::StrandBlocker')
+              .distinct
+              .where("NOT EXISTS (SELECT 1 FROM #{::Delayed::Job.quoted_table_name} dj2 WHERE delayed_jobs.strand=dj2.strand AND next_in_strand)")
+              .pluck(:strand)
+            # rubocop:enable Layout/LineLength
+            strands_to_unblock.each do |strand|
+              Delayed::Job.where(strand: strand).next_in_strand_order.first.update_attribute(:next_in_strand, true)
+            end
+          end
+        end
+      end
+
+      def migrate_everything
+        source_shard = ::Switchman::Shard.current(:delayed_jobs)
+        scope = ::Delayed::Job.shard(source_shard).where('strand IS NULL')
+
+        shard_map = build_shard_map(scope, source_shard)
+        shard_map.each do |(target_shard, source_shard_ids)|
+          batch_move_jobs(
+            target_shard: target_shard,
+            source_shard: source_shard,
+            scope: scope.where(shard_id: source_shard_ids).where(locked_by: nil)
+          )
+        end
+      end
+
+      private
+
+      def build_shard_map(scope, source_shard)
+        shard_ids = scope.distinct.pluck(:shard_id)
+
+        shard_map = {}
+        ::Switchman::Shard.find(shard_ids).each do |shard|
+          next if shard.delayed_jobs_shard == source_shard
+
+          shard_map[shard.delayed_jobs_shard] ||= []
+          shard_map[shard.delayed_jobs_shard] << shard.id
+        end
+
+        shard_map
+      end
+
+      def batch_move_jobs(target_shard:, source_shard:, scope:)
+        while scope.exists?
+          # Adapted from get_and_lock_next_available in delayed/backend/active_record.rb
+          target_jobs = scope.limit(1000).lock('FOR UPDATE SKIP LOCKED')
+
+          query = "WITH limited_jobs AS (#{target_jobs.to_sql}) " \
+                  "UPDATE #{::Delayed::Job.quoted_table_name} " \
+                  "SET locked_by = #{::Delayed::Job.connection.quote(::Delayed::Backend::Base::ON_HOLD_LOCKED_BY)}, " \
+                  "locked_at = #{::Delayed::Job.connection.quote(::Delayed::Job.db_time_now)} "\
+                  "FROM limited_jobs WHERE limited_jobs.id=#{::Delayed::Job.quoted_table_name}.id " \
+                  "RETURNING #{::Delayed::Job.quoted_table_name}.*"
+
+          jobs = source_shard.activate(:delayed_jobs) { ::Delayed::Job.find_by_sql(query) }
+          new_jobs = jobs.map do |job|
+            new_job = job.dup
+            new_job.shard = target_shard
+            new_job.created_at = job.created_at
+            new_job.updated_at = job.updated_at
+            new_job.locked_at = nil
+            new_job.locked_by = nil
+            yield(job, new_job) if block_given?
+            @before_move_callbacks&.each do |proc|
+              proc.call(
+                old_job: job,
+                new_job: new_job
+              )
+            end
+            new_job
+          end
+          transaction_on([source_shard, target_shard]) do
+            target_shard.activate(:delayed_jobs) do
+              bulk_insert_jobs(new_jobs)
+            end
+            source_shard.activate(:delayed_jobs) do
+              ::Delayed::Job.delete(jobs)
+            end
+          end
+        end
+      end
+
+      # This is adapted from the postgreql adapter in canvas-lms
+      # Once we stop supporting rails 5.2 we can just use insert_all from activerecord
+      def bulk_insert_jobs(objects)
+        records = objects.map do |object|
+          object.attributes.map do |(name, value)|
+            next if name == ::Delayed::Job.primary_key
+
+            if (type = ::Delayed::Job.attribute_types[name]).is_a?(::ActiveRecord::Type::Serialized)
+              value = type.serialize(value)
+            end
+            [name, value]
+          end.compact.to_h
+        end
+        return if records.length.zero?
+
+        keys = records.first.keys
+
+        connection = ::Delayed::Job.connection
+        quoted_keys = keys.map { |k| connection.quote_column_name(k) }.join(', ')
+
+        connection.execute "COPY #{::Delayed::Job.quoted_table_name} (#{quoted_keys}) FROM STDIN"
+        records.map do |record|
+          connection.raw_connection.put_copy_data(keys.map { |k| quote_text(record[k]) }.join("\t") + "\n")
+        end
+        connection.clear_query_cache
+        connection.raw_connection.put_copy_end
+        result = connection.raw_connection.get_result
+        begin
+          result.check
+        rescue StandardError => e
+          raise connection.send(:translate_exception, e, 'COPY FROM STDIN')
+        end
+        result.cmd_tuples
+      end
+
+      # See above comment...
+      def quote_text(value)
+        if value.nil?
+          '\\N'
+        elsif value.is_a?(::ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Array::Data)
+          quote_text(encode_array(value))
+        else
+          hash = { "\n" => '\\n', "\r" => '\\r', "\t" => '\\t', '\\' => '\\\\' }
+          value.to_s.gsub(/[\n\r\t\\]/) { |c| hash[c] }
+        end
+      end
+    end
+  end
+end
+
+# rubocop:enable Metrics/BlockLength, Metrics/MethodLength, Metrics/AbcSize, Metrics/ClassLength

data/lib/switchman_inst_jobs/switchman/default_shard.rb

@@ -4,6 +4,10 @@ module SwitchmanInstJobs
       def delayed_jobs_shard
         self
       end
+
+      def jobs_held
+        false
+      end
     end
   end
 end

data/lib/switchman_inst_jobs/switchman/shard.rb

@@ -17,6 +17,51 @@ module SwitchmanInstJobs
         database_server&.delayed_jobs_shard(self)
       end
 
+      # Adapted from hold/unhold methods in base delayed jobs base
+      # Wait is required to be able to safely move jobs
+      def hold_jobs!(wait: false)
+        self.jobs_held = true
+        save! if changed?
+        delayed_jobs_shard.activate(:delayed_jobs) do
+          lock_jobs_for_hold
+        end
+        return unless wait
+
+        delayed_jobs_shard.activate(:delayed_jobs) do
+          while ::Delayed::Job.where(shard_id: id)
+              .where.not(locked_at: nil)
+              .where.not(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY).exists?
+            sleep 10
+            lock_jobs_for_hold
+          end
+        end
+      end
+
+      def unhold_jobs!
+        self.jobs_held = false
+        save! if changed?
+        delayed_jobs_shard.activate(:delayed_jobs) do
+          ::Delayed::Job.where(locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY, shard_id: id)
+            .in_batches(of: 10_000)
+            .update_all(
+              locked_by: nil,
+              locked_at: nil,
+              attempts: 0,
+              failed_at: nil
+            )
+        end
+      end
+
+      private
+
+      def lock_jobs_for_hold
+        ::Delayed::Job.where(locked_at: nil, shard_id: id).in_batches(of: 10_000).update_all(
+          locked_by: ::Delayed::Backend::Base::ON_HOLD_LOCKED_BY,
+          locked_at: ::Delayed::Job.db_time_now,
+          attempts: ::Delayed::Backend::Base::ON_HOLD_COUNT
+        )
+      end
+
       module ClassMethods
         def clear_cache
           super
@@ -57,6 +102,14 @@ module SwitchmanInstJobs
           db.create_new_shard
         end
 
+        def periodic_clear_shard_cache
+          # TODO: make this configurable
+          @timed_cache ||= TimedCache.new(-> { 60.to_i.seconds.ago }) do
+            ::Switchman::Shard.clear_cache
+          end
+          @timed_cache.clear
+        end
+
         def delayed_jobs_shards
           unless instance_variable_defined?(:@delayed_jobs_shards)
             # re-entrancy protection

data/lib/switchman_inst_jobs/timed_cache.rb

@@ -0,0 +1,19 @@
+module SwitchmanInstJobs
+  class TimedCache
+    def initialize(timeout, &block)
+      @timeout = timeout
+      @block = block
+      @cached_at = Time.zone.now
+    end
+
+    def clear(force = false)
+      if force || @cached_at < @timeout.call
+        @block.call
+        @cached_at = Time.zone.now
+        true
+      else
+        false
+      end
+    end
+  end
+end

data/lib/switchman_inst_jobs/version.rb

@@ -1,3 +1,3 @@
 module SwitchmanInstJobs
-  VERSION = '1.4.3'.freeze
+  VERSION = '1.5.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: switchman-inst-jobs
 version: !ruby/object:Gem::Version
-  version: 1.4.3
+  version: 1.5.0
 platform: ruby
 authors:
 - Bryan Petty
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-18 00:00:00.000000000 Z
+date: 2020-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inst-jobs
@@ -16,20 +16,34 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0.16'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '0.16'
+        version: '0.17'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.15'
+        version: '0.16'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '0.16'
+        version: '0.17'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.19'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.19'
 - !ruby/object:Gem::Dependency
   name: railties
   requirement: !ruby/object:Gem::Requirement
@@ -261,6 +275,8 @@ extra_rdoc_files: []
 files:
 - db/migrate/20170308045400_add_shard_id_to_delayed_jobs.rb
 - db/migrate/20180628153808_set_search_paths_on_functions.rb
+- db/migrate/20200818130101_add_on_hold_to_switchman_shards.rb
+- db/migrate/20200822014259_add_block_stranded_to_switchman_shards.rb
 - lib/switchman-inst-jobs.rb
 - lib/switchman_inst_jobs.rb
 - lib/switchman_inst_jobs/active_record/connection_adapters/postgresql_adapter.rb
@@ -271,11 +287,13 @@ files:
 - lib/switchman_inst_jobs/delayed/worker.rb
 - lib/switchman_inst_jobs/delayed/worker/health_check.rb
 - lib/switchman_inst_jobs/engine.rb
+- lib/switchman_inst_jobs/jobs_migrator.rb
 - lib/switchman_inst_jobs/new_relic.rb
 - lib/switchman_inst_jobs/shackles.rb
 - lib/switchman_inst_jobs/switchman/database_server.rb
 - lib/switchman_inst_jobs/switchman/default_shard.rb
 - lib/switchman_inst_jobs/switchman/shard.rb
+- lib/switchman_inst_jobs/timed_cache.rb
 - lib/switchman_inst_jobs/version.rb
 - lib/switchman_inst_jobs/yaml_extensions.rb
 homepage: https://github.com/instructure/switchman-inst-jobs
@@ -297,7 +315,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Switchman and Instructure Jobs compatibility gem.