switchman-inst-jobs 4.0.3 → 4.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f07c44f5a58d897c8dbcc46a40906cde8c6a99b39723286a405669cf159019e4
-  data.tar.gz: cf730bfc3d3ad6d2c7da7712c3c097322421820b60f2d6af025ef610f0f3aa09
+  metadata.gz: 61fb55d3fff48ff95a3fae70acc49935250642ac4b2b24d3ece10e9d5624e1dd
+  data.tar.gz: 8544c358faaa63992a5db35db5d1d2a7317048349d91857d9b0241b02ec30b13
 SHA512:
-  metadata.gz: 9189978ce61d257fb25bbf57c23a36998437b3c4aa6adb93798af07ea878caf244a1514c5711c5190b39836169607e869f65b89ed71cc08d44fabfd9d6fe0e8e
-  data.tar.gz: d4a76e521ee4ba38bb2172a1ffabcc4b139a204d24690a2662ca7e8587d367ba88d98db5912c716421c96940f309296f6b35e6b292025326d61e32e03217d001
+  metadata.gz: f4601a44b1c795edb3302f5b9174292ff7724315cdb321a0fd9213f44eb72e3ffb0d48195f03426c7701eb32232b550f8bff325ea48f2f89133c7ba2e8039653
+  data.tar.gz: 8651595cec7bf1f67544e33fe806f1f082fe26809fdadacb7ad86ebfb885436db5c9333e09a1d4c320fc52dee7b1b942fe646524e1ee45d8db62b88b3f116397

data/db/migrate/20101216224513_create_delayed_jobs.rb

@@ -1,8 +1,4 @@
 class CreateDelayedJobs < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     unless connection.adapter_name == 'PostgreSQL'
       raise("#{connection.adapter_name} is not supported for delayed jobs queue")

data/db/migrate/20110208031356_add_delayed_jobs_tag.rb

@@ -1,8 +1,4 @@
 class AddDelayedJobsTag < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     add_column :delayed_jobs, :tag, :string
     add_index :delayed_jobs, [:tag]

data/db/migrate/20110426161613_add_delayed_jobs_max_attempts.rb

@@ -1,8 +1,4 @@
 class AddDelayedJobsMaxAttempts < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     add_column :delayed_jobs, :max_attempts, :integer
   end

data/db/migrate/20110516225834_add_delayed_jobs_strand.rb

@@ -1,8 +1,4 @@
 class AddDelayedJobsStrand < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     add_column :delayed_jobs, :strand, :string
     add_index :delayed_jobs, :strand

data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb

@@ -1,14 +1,10 @@
 class CleanupDelayedJobsIndexes < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     case connection.adapter_name
     when 'PostgreSQL'
       # "nulls first" syntax is postgresql specific, and allows for more
       # efficient querying for the next job
-      connection.execute("CREATE INDEX get_delayed_jobs_index ON #{::Delayed::Job.quoted_table_name} (priority, run_at, failed_at nulls first, locked_at nulls first, queue)")
+      connection.execute("CREATE INDEX get_delayed_jobs_index ON #{connection.quote_table_name(::Delayed::Job.table_name)} (priority, run_at, failed_at nulls first, locked_at nulls first, queue)")
     else
       add_index :delayed_jobs, %w[priority run_at locked_at failed_at queue], name: 'get_delayed_jobs_index'
     end

data/db/migrate/20110610213249_optimize_delayed_jobs.rb

@@ -1,8 +1,4 @@
 class OptimizeDelayedJobs < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     create_table :failed_jobs do |t|
       t.integer  'priority',    default: 0
@@ -27,11 +23,6 @@ class OptimizeDelayedJobs < ActiveRecord::Migration[4.2]
 
     add_index :delayed_jobs, %w[run_at queue locked_at strand priority], name: 'index_delayed_jobs_for_get_next'
     add_index :delayed_jobs, %w[strand id], name: 'index_delayed_jobs_on_strand'
-
-    # move all failed jobs to the new failed table
-    Delayed::Backend::ActiveRecord::Job.where.not(failed_at: nil).find_each do |job|
-      job.fail! unless job.on_hold?
-    end
   end
 
   def down

data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb

@@ -1,15 +1,11 @@
 class AddDelayedJobsNextInStrand < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     remove_index :delayed_jobs, name: 'index_delayed_jobs_for_get_next'
 
     add_column :delayed_jobs, :next_in_strand, :boolean, default: true, null: false
 
     # create the new index
-    connection.execute("CREATE INDEX get_delayed_jobs_index ON #{::Delayed::Job.quoted_table_name} (priority, run_at, queue) WHERE locked_at IS NULL AND next_in_strand = 't'")
+    connection.execute("CREATE INDEX get_delayed_jobs_index ON #{connection.quote_table_name(::Delayed::Job.table_name)} (priority, run_at, queue) WHERE locked_at IS NULL AND next_in_strand = 't'")
 
     # create the insert trigger
     execute(<<-CODE)
@@ -23,7 +19,7 @@ class AddDelayedJobsNextInStrand < ActiveRecord::Migration[4.2]
     END;
     $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
     CODE
-    execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (NEW.strand IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
+    execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{connection.quote_table_name(::Delayed::Job.table_name)} FOR EACH ROW WHEN (NEW.strand IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
 
     # create the delete trigger
     execute(<<-CODE)
@@ -34,15 +30,15 @@ class AddDelayedJobsNextInStrand < ActiveRecord::Migration[4.2]
     END;
     $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
     CODE
-    execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (OLD.strand IS NOT NULL AND OLD.next_in_strand = 't') EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} ()")
+    execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{connection.quote_table_name(::Delayed::Job.table_name)} FOR EACH ROW WHEN (OLD.strand IS NOT NULL AND OLD.next_in_strand = 't') EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} ()")
 
-    execute(%{UPDATE #{::Delayed::Job.quoted_table_name} SET next_in_strand = 'f' WHERE strand IS NOT NULL AND id <> (SELECT id FROM #{::Delayed::Job.quoted_table_name} j2 WHERE j2.strand = delayed_jobs.strand ORDER BY j2.strand, j2.id ASC LIMIT 1)})
+    execute(%{UPDATE #{connection.quote_table_name(::Delayed::Job.table_name)} SET next_in_strand = 'f' WHERE strand IS NOT NULL AND id <> (SELECT id FROM #{connection.quote_table_name(::Delayed::Job.table_name)} j2 WHERE j2.strand = delayed_jobs.strand ORDER BY j2.strand, j2.id ASC LIMIT 1)})
   end
 
   def down
-    execute %(DROP TRIGGER delayed_jobs_before_insert_row_tr ON #{::Delayed::Job.quoted_table_name})
+    execute %(DROP TRIGGER delayed_jobs_before_insert_row_tr ON #{connection.quote_table_name(::Delayed::Job.table_name)})
     execute %{DROP FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} ()}
-    execute %(DROP TRIGGER delayed_jobs_after_delete_row_tr ON #{::Delayed::Job.quoted_table_name})
+    execute %(DROP TRIGGER delayed_jobs_after_delete_row_tr ON #{connection.quote_table_name(::Delayed::Job.table_name)})
     execute %{DROP FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} ()}
 
     remove_column :delayed_jobs, :next_in_strand

data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb

@@ -1,8 +1,4 @@
 class DelayedJobsDeleteTriggerLockForUpdate < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     if connection.adapter_name == 'PostgreSQL'
       execute(<<-CODE)

data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb

@@ -1,8 +1,4 @@
 class DropPsqlJobsPopFn < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     if connection.adapter_name == 'PostgreSQL'
       connection.execute('DROP FUNCTION IF EXISTS pop_from_delayed_jobs(varchar, varchar, integer, integer, timestamp without time zone)')

data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb

@@ -1,8 +1,4 @@
 class DelayedJobsUseAdvisoryLocks < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     # use an advisory lock based on the name of the strand, instead of locking the whole table
     # note that we're using half of the md5, so collisions are possible, but we don't really

data/db/migrate/20120607181141_index_jobs_on_locked_by.rb

@@ -1,10 +1,6 @@
 class IndexJobsOnLockedBy < ActiveRecord::Migration[4.2]
   disable_ddl_transaction!
 
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     add_index :delayed_jobs, :locked_by, algorithm: :concurrently, where: 'locked_by IS NOT NULL'
   end

data/db/migrate/20120608191051_add_jobs_run_at_index.rb

@@ -1,10 +1,6 @@
 class AddJobsRunAtIndex < ActiveRecord::Migration[4.2]
   disable_ddl_transaction!
 
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     add_index :delayed_jobs, %w[run_at tag], algorithm: :concurrently
   end

data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb

@@ -1,8 +1,4 @@
 class ChangeDelayedJobsHandlerToText < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     change_column :delayed_jobs, :handler, :text
   end

data/db/migrate/20140505215131_add_failed_jobs_original_job_id.rb

@@ -1,8 +1,4 @@
 class AddFailedJobsOriginalJobId < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     add_column :failed_jobs, :original_job_id, :integer, limit: 8
   end

data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb

@@ -1,11 +1,7 @@
 class CopyFailedJobsOriginalId < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
-    # this is a smaller, less frequently accessed table, so we just update all at once
-    Delayed::Backend::ActiveRecord::Job::Failed.where(original_job_id: nil).update_all('original_job_id = original_id')
+    # Noop since we don't want to modify the shard using a different connection than the one we are using to build it and
+    # this migration is very old
   end
 
   def down; end

data/db/migrate/20140505223637_drop_failed_jobs_original_id.rb

@@ -1,8 +1,4 @@
 class DropFailedJobsOriginalId < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     remove_column :failed_jobs, :original_id
   end

data/db/migrate/20140512213941_add_source_to_jobs.rb

@@ -1,8 +1,4 @@
 class AddSourceToJobs < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     add_column :delayed_jobs, :source, :string
     add_column :failed_jobs, :source, :string

data/db/migrate/20150807133223_add_max_concurrent_to_jobs.rb

@@ -1,8 +1,4 @@
 class AddMaxConcurrentToJobs < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     add_column :delayed_jobs, :max_concurrent, :integer, default: 1, null: false
 

data/db/migrate/20151123210429_add_expires_at_to_jobs.rb

@@ -1,8 +1,4 @@
 class AddExpiresAtToJobs < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     add_column :delayed_jobs, :expires_at, :datetime
     add_column :failed_jobs, :expires_at, :datetime

data/db/migrate/20151210162949_improve_max_concurrent.rb

@@ -1,8 +1,4 @@
 class ImproveMaxConcurrent < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     if connection.adapter_name == 'PostgreSQL'
       execute(<<-CODE)

data/db/migrate/20161206323555_add_back_default_string_limits_jobs.rb

@@ -1,8 +1,4 @@
 class AddBackDefaultStringLimitsJobs < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     drop_triggers
 
@@ -22,13 +18,13 @@ class AddBackDefaultStringLimitsJobs < ActiveRecord::Migration[4.2]
   end
 
   def drop_triggers
-    execute %(DROP TRIGGER delayed_jobs_before_insert_row_tr ON #{::Delayed::Job.quoted_table_name})
-    execute %(DROP TRIGGER delayed_jobs_after_delete_row_tr ON #{::Delayed::Job.quoted_table_name})
+    execute %(DROP TRIGGER delayed_jobs_before_insert_row_tr ON #{connection.quote_table_name(::Delayed::Job.table_name)})
+    execute %(DROP TRIGGER delayed_jobs_after_delete_row_tr ON #{connection.quote_table_name(::Delayed::Job.table_name)})
   end
 
   def readd_triggers
-    execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (NEW.strand IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
-    execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (OLD.strand IS NOT NULL AND OLD.next_in_strand = 't') EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')}()")
+    execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{connection.quote_table_name(::Delayed::Job.table_name)} FOR EACH ROW WHEN (NEW.strand IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
+    execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{connection.quote_table_name(::Delayed::Job.table_name)} FOR EACH ROW WHEN (OLD.strand IS NOT NULL AND OLD.next_in_strand = 't') EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')}()")
   end
 
   def add_string_limit_if_missing(table, column)

data/db/migrate/20170308045400_add_shard_id_to_delayed_jobs.rb

@@ -1,10 +1,6 @@
 class AddShardIdToDelayedJobs < ActiveRecord::Migration[4.2]
   disable_ddl_transaction!
 
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     add_column :delayed_jobs, :shard_id, :integer, limit: 8
     add_index :delayed_jobs, :shard_id, algorithm: :concurrently

data/db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb

@@ -1,8 +1,4 @@
 class SpeedUpMaxConcurrentTriggers < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     if connection.adapter_name == 'PostgreSQL'
       # tl;dr sacrifice some responsiveness to max_concurrent changes for faster performance

data/db/migrate/20190726154743_make_critical_columns_not_null.rb

@@ -1,8 +1,4 @@
 class MakeCriticalColumnsNotNull < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     change_column_null :delayed_jobs, :run_at, false
     change_column_null :delayed_jobs, :queue, false

data/db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb

@@ -1,10 +1,6 @@
 class AddIdToGetDelayedJobsIndex < ActiveRecord::Migration[4.2]
   disable_ddl_transaction!
 
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     rename_index :delayed_jobs, 'get_delayed_jobs_index', 'get_delayed_jobs_index_old'
     add_index :delayed_jobs, %i[queue priority run_at id],

data/db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb

@@ -1,8 +1,4 @@
 class SpeedUpMaxConcurrentDeleteTrigger < ActiveRecord::Migration[4.2]
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     if connection.adapter_name == 'PostgreSQL'
       # tl;dr sacrifice some responsiveness to max_concurrent changes for faster performance

data/db/migrate/20200825011002_add_strand_order_override.rb

@@ -1,10 +1,6 @@
 class AddStrandOrderOverride < ActiveRecord::Migration[4.2]
   disable_ddl_transaction!
 
-  def connection
-    Delayed::Backend::ActiveRecord::AbstractJob.connection
-  end
-
   def up
     add_column :delayed_jobs, :strand_order_override, :integer, default: 0, null: false
     add_column :failed_jobs, :strand_order_override, :integer, default: 0, null: false

data/db/migrate/20210812210128_add_singleton_column.rb

@@ -181,20 +181,20 @@ class AddSingletonColumn < ActiveRecord::Migration[5.2]
       reversible do |direction|
         direction.up do
           drop_triggers
-          execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (NEW.strand IS NOT NULL OR NEW.singleton IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
-          execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN ((OLD.strand IS NOT NULL OR OLD.singleton IS NOT NULL) AND OLD.next_in_strand=true) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')}()")
+          execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{connection.quote_table_name(::Delayed::Job.table_name)} FOR EACH ROW WHEN (NEW.strand IS NOT NULL OR NEW.singleton IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
+          execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{connection.quote_table_name(::Delayed::Job.table_name)} FOR EACH ROW WHEN ((OLD.strand IS NOT NULL OR OLD.singleton IS NOT NULL) AND OLD.next_in_strand=true) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')}()")
         end
         direction.down do
           drop_triggers
-          execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (NEW.strand IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
-          execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (OLD.strand IS NOT NULL AND OLD.next_in_strand = 't') EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn()')}")
+          execute("CREATE TRIGGER delayed_jobs_before_insert_row_tr BEFORE INSERT ON #{connection.quote_table_name(::Delayed::Job.table_name)} FOR EACH ROW WHEN (NEW.strand IS NOT NULL) EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')}()")
+          execute("CREATE TRIGGER delayed_jobs_after_delete_row_tr AFTER DELETE ON #{connection.quote_table_name(::Delayed::Job.table_name)} FOR EACH ROW WHEN (OLD.strand IS NOT NULL AND OLD.next_in_strand = 't') EXECUTE PROCEDURE #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn()')}")
         end
       end
     end
   end
 
   def drop_triggers
-    execute("DROP TRIGGER delayed_jobs_before_insert_row_tr ON #{::Delayed::Job.quoted_table_name}")
-    execute("DROP TRIGGER delayed_jobs_after_delete_row_tr ON #{::Delayed::Job.quoted_table_name}")
+    execute("DROP TRIGGER delayed_jobs_before_insert_row_tr ON #{connection.quote_table_name(::Delayed::Job.table_name)}")
+    execute("DROP TRIGGER delayed_jobs_after_delete_row_tr ON #{connection.quote_table_name(::Delayed::Job.table_name)}")
   end
 end

data/db/migrate/20210917232626_add_delete_conflicting_singletons_before_unlock_trigger.rb

@@ -13,7 +13,7 @@ class AddDeleteConflictingSingletonsBeforeUnlockTrigger < ActiveRecord::Migratio
       $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
     SQL
     execute(<<~SQL)
-      CREATE TRIGGER delayed_jobs_before_unlock_delete_conflicting_singletons_row_tr BEFORE UPDATE ON #{::Delayed::Job.quoted_table_name} FOR EACH ROW WHEN (
+      CREATE TRIGGER delayed_jobs_before_unlock_delete_conflicting_singletons_row_tr BEFORE UPDATE ON #{connection.quote_table_name(::Delayed::Job.table_name)} FOR EACH ROW WHEN (
         OLD.singleton IS NOT NULL AND
         OLD.singleton=NEW.singleton AND
         OLD.locked_by IS NOT NULL AND

data/db/migrate/20220127091200_fix_singleton_unique_constraint.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class FixSingletonUniqueConstraint < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    rename_index :delayed_jobs, 'index_delayed_jobs_on_singleton_not_running', 'index_delayed_jobs_on_singleton_not_running_old'
+    rename_index :delayed_jobs, 'index_delayed_jobs_on_singleton_running', 'index_delayed_jobs_on_singleton_running_old'
+
+    # only one job can be queued in a singleton
+    add_index :delayed_jobs,
+              :singleton,
+              where: "singleton IS NOT NULL AND (locked_by IS NULL OR locked_by = '#{::Delayed::Backend::Base::ON_HOLD_LOCKED_BY}')",
+              unique: true,
+              name: 'index_delayed_jobs_on_singleton_not_running',
+              algorithm: :concurrently
+
+    # only one job can be running for a singleton
+    add_index :delayed_jobs,
+              :singleton,
+              where: "singleton IS NOT NULL AND locked_by IS NOT NULL AND locked_by <> '#{::Delayed::Backend::Base::ON_HOLD_LOCKED_BY}'",
+              unique: true,
+              name: 'index_delayed_jobs_on_singleton_running',
+              algorithm: :concurrently
+  end
+
+  def down
+    remove_index :delayed_jobs, name: 'index_delayed_jobs_on_singleton_not_running_old'
+    remove_index :delayed_jobs, name: 'index_delayed_jobs_on_singleton_running_old'
+  end
+end

data/db/migrate/20220128084800_update_insert_trigger_for_singleton_unique_constraint_change.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+class UpdateInsertTriggerForSingletonUniqueConstraintChange < ActiveRecord::Migration[5.2]
+  def change
+    reversible do |direction|
+      direction.up do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(CONCAT('singleton:', NEW.singleton)));
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by = '#{::Delayed::Backend::Base::ON_HOLD_LOCKED_BY}' OR locked_by <> '#{::Delayed::Backend::Base::ON_HOLD_LOCKED_BY}');
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+      end
+      direction.down do
+        execute(<<~SQL)
+          CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_before_insert_row_tr_fn')} () RETURNS trigger AS $$
+          BEGIN
+            IF NEW.strand IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(NEW.strand));
+              IF (SELECT COUNT(*) FROM (
+                  SELECT 1 FROM delayed_jobs WHERE strand = NEW.strand AND next_in_strand=true LIMIT NEW.max_concurrent
+                ) s) = NEW.max_concurrent THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            IF NEW.singleton IS NOT NULL THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(CONCAT('singleton:', NEW.singleton)));
+              -- this condition seems silly, but it forces postgres to use the two partial indexes on singleton,
+              -- rather than doing a seq scan
+              PERFORM 1 FROM delayed_jobs WHERE singleton = NEW.singleton AND (locked_by IS NULL OR locked_by IS NOT NULL);
+              IF FOUND THEN
+                NEW.next_in_strand := false;
+              END IF;
+            END IF;
+            RETURN NEW;
+          END;
+          $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+        SQL
+      end
+    end
+  end
+end

data/db/migrate/20220128084900_update_delete_trigger_for_singleton_unique_constraint_change.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+
+class UpdateDeleteTriggerForSingletonUniqueConstraintChange < ActiveRecord::Migration[5.2]
+  def up
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+      DECLARE
+        next_strand varchar;
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+        transition boolean;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by = ''#{::Delayed::Backend::Base::ON_HOLD_LOCKED_BY}'' OR j3.locked_by <> ''#{::Delayed::Backend::Base::ON_HOLD_LOCKED_BY}'')))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        END IF;
+
+        IF OLD.singleton IS NOT NULL THEN
+          PERFORM pg_advisory_xact_lock(half_md5_as_bigint(CONCAT('singleton:', OLD.singleton)));
+
+          transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
+
+          IF transition THEN
+            next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
+
+            IF next_strand IS NOT NULL THEN
+              -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
+              IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
+                -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
+                RETURN OLD;
+              END IF;
+            END IF;
+          ELSIF OLD.strand IS NOT NULL THEN
+            -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
+            RETURN OLD;
+          END IF;
+
+          -- handles transitioning a singleton from stranded to not stranded --
+          -- handles transitioning a singleton from unstranded to stranded --
+          -- handles transitioning a singleton from strand A to strand B --
+          -- these transitions are a relatively rare case, so we take a shortcut and --
+          -- only start the next singleton if its strand does not currently have any running jobs --
+          -- if it does, the next stranded job that finishes will start this singleton if it can --
+          UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.singleton=OLD.singleton AND
+                j2.locked_by IS NULL AND
+                (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
+              FOR UPDATE
+            );
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+
+  def down
+    execute(<<~SQL)
+      CREATE OR REPLACE FUNCTION #{connection.quote_table_name('delayed_jobs_after_delete_row_tr_fn')} () RETURNS trigger AS $$
+      DECLARE
+        next_strand varchar;
+        running_count integer;
+        should_lock boolean;
+        should_be_precise boolean;
+        update_query varchar;
+        skip_locked varchar;
+        transition boolean;
+      BEGIN
+        IF OLD.strand IS NOT NULL THEN
+          should_lock := true;
+          should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+
+          IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) subquery_for_count);
+            should_lock := running_count < OLD.max_concurrent;
+          END IF;
+
+          IF should_lock THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+          END IF;
+
+          -- note that we don't really care if the row we're deleting has a singleton, or if it even
+          -- matches the row(s) we're going to update. we just need to make sure that whatever
+          -- singleton we grab isn't already running (which is a simple existence check, since
+          -- the unique indexes ensure there is at most one singleton running, and one queued)
+          update_query := 'UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.strand=$1.strand AND
+                (j2.singleton IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.singleton=j2.singleton AND j3.id<>j2.id AND (j3.locked_by IS NULL OR j3.locked_by IS NOT NULL)))
+              ORDER BY j2.strand_order_override ASC, j2.id ASC
+              LIMIT ';
+
+          IF should_be_precise THEN
+            running_count := (SELECT COUNT(*) FROM (
+              SELECT 1 FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+            ) s);
+            IF running_count < OLD.max_concurrent THEN
+              update_query := update_query || '($1.max_concurrent - $2)';
+            ELSE
+              -- we have too many running already; just bail
+              RETURN OLD;
+            END IF;
+          ELSE
+            update_query := update_query || '1';
+
+            -- n-strands don't require precise ordering; we can make this query more performant
+            IF OLD.max_concurrent > 1 THEN
+              skip_locked := ' SKIP LOCKED';
+            END IF;
+          END IF;
+
+          update_query := update_query || ' FOR UPDATE' || COALESCE(skip_locked, '') || ')';
+          EXECUTE update_query USING OLD, running_count;
+        END IF;
+
+        IF OLD.singleton IS NOT NULL THEN
+          PERFORM pg_advisory_xact_lock(half_md5_as_bigint(CONCAT('singleton:', OLD.singleton)));
+
+          transition := EXISTS (SELECT 1 FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL);
+
+          IF transition THEN
+            next_strand := (SELECT j1.strand FROM delayed_jobs AS j1 WHERE j1.singleton = OLD.singleton AND j1.strand IS DISTINCT FROM OLD.strand AND locked_by IS NULL AND j1.strand IS NOT NULL LIMIT 1);
+
+            IF next_strand IS NOT NULL THEN
+              -- if the singleton has a new strand defined, we need to lock it to ensure we obey n_strand constraints --
+              IF NOT pg_try_advisory_xact_lock(half_md5_as_bigint(next_strand)) THEN
+                -- a failure to acquire the lock means that another process already has it and will thus handle this singleton --
+                RETURN OLD;
+              END IF;
+            END IF;
+          ELSIF OLD.strand IS NOT NULL THEN
+            -- if there is no transition and there is a strand then we have already handled this singleton in the case above --
+            RETURN OLD;
+          END IF;
+
+          -- handles transitioning a singleton from stranded to not stranded --
+          -- handles transitioning a singleton from unstranded to stranded --
+          -- handles transitioning a singleton from strand A to strand B --
+          -- these transitions are a relatively rare case, so we take a shortcut and --
+          -- only start the next singleton if its strand does not currently have any running jobs --
+          -- if it does, the next stranded job that finishes will start this singleton if it can --
+          UPDATE delayed_jobs SET next_in_strand=true WHERE id IN (
+            SELECT id FROM delayed_jobs j2
+              WHERE next_in_strand=false AND
+                j2.singleton=OLD.singleton AND
+                j2.locked_by IS NULL AND
+                (j2.strand IS NULL OR NOT EXISTS (SELECT 1 FROM delayed_jobs j3 WHERE j3.strand=j2.strand AND j3.id<>j2.id))
+              FOR UPDATE
+            );
+        END IF;
+        RETURN OLD;
+      END;
+      $$ LANGUAGE plpgsql SET search_path TO #{::Switchman::Shard.current.name};
+    SQL
+  end
+end

data/db/migrate/20220203063200_remove_old_singleton_index.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class RemoveOldSingletonIndex < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    remove_index :delayed_jobs, name: 'index_delayed_jobs_on_singleton_not_running_old'
+    remove_index :delayed_jobs, name: 'index_delayed_jobs_on_singleton_running_old'
+  end
+
+  def down
+    rename_index :delayed_jobs, 'index_delayed_jobs_on_singleton_not_running', 'index_delayed_jobs_on_singleton_not_running_old'
+    rename_index :delayed_jobs, 'index_delayed_jobs_on_singleton_running', 'index_delayed_jobs_on_singleton_running_old'
+
+    # only one job can be queued in a singleton
+    add_index :delayed_jobs,
+              :singleton,
+              where: 'singleton IS NOT NULL AND locked_by IS NULL',
+              unique: true,
+              name: 'index_delayed_jobs_on_singleton_not_running',
+              algorithm: :concurrently
+
+    # only one job can be running for a singleton
+    add_index :delayed_jobs,
+              :singleton,
+              where: 'singleton IS NOT NULL AND locked_by IS NOT NULL',
+              unique: true,
+              name: 'index_delayed_jobs_on_singleton_running',
+              algorithm: :concurrently
+  end
+end

data/lib/switchman_inst_jobs/delayed/backend/active_record/abstract_job.rb

@@ -0,0 +1,25 @@
+module SwitchmanInstJobs
+  module Delayed
+    module Backend
+      module ActiveRecord
+        module AbstractJob
+          module ClassMethods
+            def current_switchman_shard
+              connected_to_stack.reverse_each do |hash|
+                return hash[:switchman_shard] if hash[:switchman_shard] && hash[:klasses].include?(connection_classes)
+              end
+
+              ::ActiveRecord::Base.current_switchman_shard.delayed_jobs_shard
+            end
+          end
+
+          def self.prepended(base)
+            base.singleton_class.prepend(ClassMethods)
+
+            base.sharded_model
+          end
+        end
+      end
+    end
+  end
+end

data/lib/switchman_inst_jobs/engine.rb

@@ -21,13 +21,17 @@ module SwitchmanInstJobs
 
       # Ensure jobs get unblocked on the new shard if they exist
       ::Delayed::Worker.lifecycle.after(:perform) do |_worker, job|
-        if job.strand
+        if job.strand || job.singleton
+          column = job.strand ? :strand : :singleton
+
           ::Switchman::Shard.clear_cache
           ::Switchman::Shard.default.activate do
             current_job_shard = ::Switchman::Shard.lookup(job.shard_id).delayed_jobs_shard
             if current_job_shard != ::Switchman::Shard.current(::Delayed::Backend::ActiveRecord::AbstractJob)
               current_job_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
-                j = ::Delayed::Job.where(strand: job.strand).next_in_strand_order.first
+                ::Delayed::Job.where(source: 'JobsMigrator::StrandBlocker', **{ column => job.try(column) }).delete_all
+
+                j = ::Delayed::Job.where(**{ column => job.try(column) }).next_in_strand_order.first
                 j.update_column(:next_in_strand, true) if j && !j.next_in_strand
               end
             end

data/lib/switchman_inst_jobs/jobs_migrator.rb

@@ -89,7 +89,9 @@ module SwitchmanInstJobs
         migrate_everything
       end
 
-      def migrate_strands
+      def migrate_strands(batch_size: 1_000)
+        source_shard = ::Switchman::Shard.current(::Delayed::Backend::ActiveRecord::AbstractJob)
+
         # there are 4 scenarios to deal with here
         # 1) no running job, no jobs moved: do nothing
         # 2) running job, no jobs moved; create blocker with next_in_strand=false
@@ -98,60 +100,89 @@ module SwitchmanInstJobs
         #    those (= do nothing since it should already be false)
         # 4) no running job, jobs moved: set next_in_strand=true on the first of
         #    those (= do nothing since it should already be true)
+        handler = lambda { |scope, column, blocker_job_kwargs = {}, advisory_lock_cb = nil|
+          shard_map = build_shard_map(scope, source_shard)
+          shard_map.each do |(target_shard, source_shard_ids)|
+            shard_scope = scope.where(shard_id: source_shard_ids)
 
-        source_shard = ::Switchman::Shard.current(::Delayed::Backend::ActiveRecord::AbstractJob)
-        strand_scope = ::Delayed::Job.shard(source_shard).where.not(strand: nil)
-        shard_map = build_shard_map(strand_scope, source_shard)
-        shard_map.each do |(target_shard, source_shard_ids)|
-          shard_scope = strand_scope.where(shard_id: source_shard_ids)
-
-          # 1) is taken care of because it should not show up here in strands
-          strands = shard_scope.distinct.order(:strand).pluck(:strand)
-
-          target_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
-            strands.each do |strand|
-              transaction_on([source_shard, target_shard]) do
-                this_strand_scope = shard_scope.where(strand: strand)
-                # we want to copy all the jobs except the one that is still running.
-                jobs_scope = this_strand_scope.where(locked_by: nil)
-
-                # 2) and part of 3) are taken care of here by creating a blocker
-                # job with next_in_strand = false. as soon as the current
-                # running job is finished it should set next_in_strand
-                # We lock it to ensure that the jobs worker can't delete it until we are done moving the strand
-                # Since we only unlock it on the new jobs queue *after* deleting from the original
-                # the lock ensures the blocker always gets unlocked
-                first = this_strand_scope.where.not(locked_by: nil).next_in_strand_order.lock.first
-                if first
-                  first_job = ::Delayed::Job.create!(strand: strand, next_in_strand: false)
-                  first_job.payload_object = ::Delayed::PerformableMethod.new(Kernel, :sleep, args: [0])
-                  first_job.queue = first.queue
-                  first_job.tag = 'Kernel.sleep'
-                  first_job.source = 'JobsMigrator::StrandBlocker'
-                  first_job.max_attempts = 1
-                  # If we ever have jobs left over from 9999 jobs moves of a single shard,
-                  # something has gone terribly wrong
-                  first_job.strand_order_override = -9999
-                  first_job.save!
-                  # the rest of 3) is taken care of here
-                  # make sure that all the jobs moved over are NOT next in strand
-                  ::Delayed::Job.where(next_in_strand: true, strand: strand, locked_by: nil).
-                    update_all(next_in_strand: false)
-                end
+            # 1) is taken care of because it should not show up here in strands
+            values = shard_scope.distinct.order(column).pluck(column)
 
-                # 4) is taken care of here, by leaving next_in_strand alone and
-                # it should execute on the new shard
-                batch_move_jobs(
-                  target_shard: target_shard,
-                  source_shard: source_shard,
-                  scope: jobs_scope
-                ) do |job, new_job|
-                  # This ensures jobs enqueued on the old jobs shard run before jobs on the new jobs queue
-                  new_job.strand_order_override = job.strand_order_override - 1
+            target_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
+              values.each do |value|
+                transaction_on([source_shard, target_shard]) do
+                  source_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
+                    advisory_lock_cb&.call(value)
+                  end
+
+                  value_scope = shard_scope.where(**{ column => value })
+                  # we want to copy all the jobs except the one that is still running.
+                  jobs_scope = value_scope.where(locked_by: nil)
+
+                  # 2) and part of 3) are taken care of here by creating a blocker
+                  # job with next_in_strand = false. as soon as the current
+                  # running job is finished it should set next_in_strand
+                  # We lock it to ensure that the jobs worker can't delete it until we are done moving the strand
+                  # Since we only unlock it on the new jobs queue *after* deleting from the original
+                  # the lock ensures the blocker always gets unlocked
+                  first = value_scope.where.not(locked_by: nil).next_in_strand_order.lock.first
+                  if first
+                    create_blocker_job(
+                      queue: first.queue,
+                      shard_id: first.shard_id,
+                      **{ column => value },
+                      **blocker_job_kwargs
+                    )
+
+                    # the rest of 3) is taken care of here
+                    # make sure that all the jobs moved over are NOT next in strand
+                    ::Delayed::Job.where(next_in_strand: true, locked_by: nil, **{ column => value }).
+                      update_all(next_in_strand: false)
+                  end
+
+                  # 4) is taken care of here, by leaving next_in_strand alone and
+                  # it should execute on the new shard
+                  batch_move_jobs(
+                    target_shard: target_shard,
+                    source_shard: source_shard,
+                    scope: jobs_scope,
+                    batch_size: batch_size
+                  ) do |job, new_job|
+                    # This ensures jobs enqueued on the old jobs shard run before jobs on the new jobs queue
+                    new_job.strand_order_override = job.strand_order_override - 1
+                  end
                 end
               end
             end
+          end
+        }
+
+        strand_scope = ::Delayed::Job.shard(source_shard).where.not(strand: nil)
+        singleton_scope = ::Delayed::Job.shard(source_shard).where('strand IS NULL AND singleton IS NOT NULL')
+        all_scope = ::Delayed::Job.shard(source_shard).where('strand IS NOT NULL OR singleton IS NOT NULL')
 
+        singleton_blocker_additional_kwargs = {
+          locked_at: DateTime.now,
+          locked_by: ::Delayed::Backend::Base::ON_HOLD_BLOCKER
+        }
+
+        quoted_function_name = ::Delayed::Job.connection.quote_table_name('half_md5_as_bigint')
+        strand_advisory_lock_fn = lambda do |value|
+          ::Delayed::Job.connection.execute("SELECT pg_advisory_xact_lock(#{quoted_function_name}('#{value}'))")
+        end
+
+        singleton_advisory_lock_fn = lambda do |value|
+          ::Delayed::Job.connection.execute(
+            "SELECT pg_advisory_xact_lock(#{quoted_function_name}('singleton:#{value}'))"
+          )
+        end
+
+        handler.call(strand_scope, :strand, {}, strand_advisory_lock_fn)
+        handler.call(singleton_scope, :singleton, singleton_blocker_additional_kwargs, singleton_advisory_lock_fn)
+
+        shard_map = build_shard_map(all_scope, source_shard)
+        shard_map.each do |(target_shard, source_shard_ids)|
+          target_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
             updated = ::Switchman::Shard.where(id: source_shard_ids, block_stranded: true).
               update_all(block_stranded: false)
             # If this is being manually re-run for some reason to clean something up, don't wait for nothing to happen
@@ -166,26 +197,40 @@ module SwitchmanInstJobs
         end
       end
 
-      def unblock_strands(target_shard)
-        target_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
-          loop do
-            # We only want to unlock stranded jobs where they don't belong to a blocked shard (if they *do* belong)
-            # to a blocked shard, they must be part of a concurrent jobs migration from a different source shard to
-            # this target shard, so we shouldn't unlock them yet.  We only ever unlock one job here to keep the
-            # logic cleaner; if the job is n-stranded, after the first one runs, the trigger will unlock larger
-            # batches
-            break if ::Delayed::Job.where(id: ::Delayed::Job.select('DISTINCT ON (strand) id').
-              where.not(strand: nil).
-              where.not(shard_id: ::Switchman::Shard.where(block_stranded: true).pluck(:id)).where(
+      def unblock_strands(target_shard, batch_size: 10_000)
+        block_stranded_ids = ::Switchman::Shard.where(block_stranded: true).pluck(:id)
+        query = lambda { |column, scope|
+          ::Delayed::Job.
+            where(id: ::Delayed::Job.select("DISTINCT ON (#{column}) id").
+              where(scope).
+              where.not(shard_id: block_stranded_ids).
+              where(
                 ::Delayed::Job.select(1).from("#{::Delayed::Job.quoted_table_name} dj2").
                 where("dj2.next_in_strand = true OR dj2.source = 'JobsMigrator::StrandBlocker'").
-                where('dj2.strand = delayed_jobs.strand').arel.exists.not
-              ).order(:strand, :strand_order_override, :id)).limit(500).update_all(next_in_strand: true).zero?
+                where("dj2.#{column} = delayed_jobs.#{column}").arel.exists.not
+              ).
+              order(column, :strand_order_override, :id)).limit(batch_size)
+        }
+
+        target_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
+          # We only want to unlock stranded jobs where they don't belong to a blocked shard (if they *do* belong)
+          # to a blocked shard, they must be part of a concurrent jobs migration from a different source shard to
+          # this target shard, so we shouldn't unlock them yet.  We only ever unlock one job here to keep the
+          # logic cleaner; if the job is n-stranded, after the first one runs, the trigger will unlock larger
+          # batches
+
+          loop do
+            break if query.call(:strand, 'strand IS NOT NULL').update_all(next_in_strand: true).zero?
+          end
+
+          loop do
+            break if query.call(:singleton,
+                                'strand IS NULL AND singleton IS NOT NULL').update_all(next_in_strand: true).zero?
           end
         end
       end
 
-      def migrate_everything
+      def migrate_everything(batch_size: 1_000)
         source_shard = ::Switchman::Shard.current(::Delayed::Backend::ActiveRecord::AbstractJob)
         scope = ::Delayed::Job.shard(source_shard).where(strand: nil)
 
@@ -194,13 +239,26 @@ module SwitchmanInstJobs
           batch_move_jobs(
             target_shard: target_shard,
             source_shard: source_shard,
-            scope: scope.where(shard_id: source_shard_ids).where(locked_by: nil)
+            scope: scope.where(shard_id: source_shard_ids).where(locked_by: nil),
+            batch_size: batch_size
           )
         end
       end
 
       private
 
+      def create_blocker_job(**kwargs)
+        first_job = ::Delayed::Job.create!(**kwargs, next_in_strand: false)
+        first_job.payload_object = ::Delayed::PerformableMethod.new(Kernel, :sleep, args: [0])
+        first_job.tag = 'Kernel.sleep'
+        first_job.source = 'JobsMigrator::StrandBlocker'
+        first_job.max_attempts = 1
+        # If we ever have jobs left over from 9999 jobs moves of a single shard,
+        # something has gone terribly wrong
+        first_job.strand_order_override = -9999
+        first_job.save!
+      end
+
       def build_shard_map(scope, source_shard)
         shard_ids = scope.distinct.pluck(:shard_id)
 
@@ -215,10 +273,10 @@ module SwitchmanInstJobs
         shard_map
       end
 
-      def batch_move_jobs(target_shard:, source_shard:, scope:)
+      def batch_move_jobs(target_shard:, source_shard:, scope:, batch_size:)
         while scope.exists?
           # Adapted from get_and_lock_next_available in delayed/backend/active_record.rb
-          target_jobs = scope.limit(1000).lock('FOR UPDATE SKIP LOCKED')
+          target_jobs = scope.limit(batch_size).lock('FOR UPDATE SKIP LOCKED')
 
           query = source_shard.activate(::Delayed::Backend::ActiveRecord::AbstractJob) do
             <<~SQL

data/lib/switchman_inst_jobs/version.rb

@@ -1,3 +1,3 @@
 module SwitchmanInstJobs
-  VERSION = '4.0.3'.freeze
+  VERSION = '4.0.6'.freeze
 end

data/lib/switchman_inst_jobs.rb

@@ -5,9 +5,6 @@ module SwitchmanInstJobs
   cattr_accessor :delayed_jobs_shard_fallback
 
   def self.initialize_active_record
-    ::ActiveRecord::ConnectionAdapters::ConnectionPool.prepend(
-      ActiveRecord::ConnectionAdapters::ConnectionPool
-    )
     ::ActiveRecord::ConnectionAdapters::PostgreSQLAdapter.prepend(
       ActiveRecord::ConnectionAdapters::PostgreSQLAdapter
     )
@@ -17,6 +14,9 @@ module SwitchmanInstJobs
     ::Delayed::Backend::ActiveRecord::Job.prepend(
       Delayed::Backend::Base
     )
+    ::Delayed::Backend::ActiveRecord::AbstractJob.prepend(
+      Delayed::Backend::ActiveRecord::AbstractJob
+    )
     ::Delayed::Pool.prepend Delayed::Pool
     ::Delayed::Worker.prepend Delayed::Worker
     ::Delayed::Worker::HealthCheck.prepend Delayed::Worker::HealthCheck
@@ -35,10 +35,10 @@ module SwitchmanInstJobs
   end
 end
 
-require 'switchman_inst_jobs/active_record/connection_adapters/connection_pool'
 require 'switchman_inst_jobs/active_record/connection_adapters/postgresql_adapter'
 require 'switchman_inst_jobs/active_record/migration'
 require 'switchman_inst_jobs/delayed/settings'
+require 'switchman_inst_jobs/delayed/backend/active_record/abstract_job'
 require 'switchman_inst_jobs/delayed/backend/base'
 require 'switchman_inst_jobs/delayed/message_sending'
 require 'switchman_inst_jobs/delayed/pool'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: switchman-inst-jobs
 version: !ruby/object:Gem::Version
-  version: 4.0.3
+  version: 4.0.6
 platform: ruby
 authors:
 - Bryan Petty
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-12-20 00:00:00.000000000 Z
+date: 2022-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: inst-jobs
@@ -73,7 +73,7 @@ dependencies:
         version: '3.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.1
+        version: 3.0.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -83,7 +83,7 @@ dependencies:
         version: '3.0'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.0.1
+        version: 3.0.7
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -340,11 +340,15 @@ files:
 - db/migrate/20211207094200_update_after_delete_trigger_for_singleton_transition_cases.rb
 - db/migrate/20211220112800_fix_singleton_race_condition_insert.rb
 - db/migrate/20211220113000_fix_singleton_race_condition_delete.rb
+- db/migrate/20220127091200_fix_singleton_unique_constraint.rb
+- db/migrate/20220128084800_update_insert_trigger_for_singleton_unique_constraint_change.rb
+- db/migrate/20220128084900_update_delete_trigger_for_singleton_unique_constraint_change.rb
+- db/migrate/20220203063200_remove_old_singleton_index.rb
 - lib/switchman-inst-jobs.rb
 - lib/switchman_inst_jobs.rb
-- lib/switchman_inst_jobs/active_record/connection_adapters/connection_pool.rb
 - lib/switchman_inst_jobs/active_record/connection_adapters/postgresql_adapter.rb
 - lib/switchman_inst_jobs/active_record/migration.rb
+- lib/switchman_inst_jobs/delayed/backend/active_record/abstract_job.rb
 - lib/switchman_inst_jobs/delayed/backend/base.rb
 - lib/switchman_inst_jobs/delayed/message_sending.rb
 - lib/switchman_inst_jobs/delayed/pool.rb

data/lib/switchman_inst_jobs/active_record/connection_adapters/connection_pool.rb

@@ -1,15 +0,0 @@
-module SwitchmanInstJobs
-  module ActiveRecord
-    module ConnectionAdapters
-      module ConnectionPool
-        def shard
-          if connection_klass == ::Delayed::Backend::ActiveRecord::AbstractJob
-            return shard_stack.last || ::Switchman::Shard.current(::ActiveRecord::Base).delayed_jobs_shard
-          end
-
-          super
-        end
-      end
-    end
-  end
-end