inst-jobs 0.15.22 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 33b7f135ba67d0b2b056851cec2e95672ac03766fa48d614ef53fa0e25a04255
-  data.tar.gz: 29878f6688d376811085f7615c701ae2f7cf31cc3bf6462440823f4e91ccb271
+  metadata.gz: 78a0627c587b4c10c24df960b67b4a6427fe5b9c8e9561f0464864203a43c5e9
+  data.tar.gz: 0deba8f386d28954fe5b196aa86551d3bc50d6b98d261ffc3468406645a464d5
 SHA512:
-  metadata.gz: 8695ff4bd7e9c10a5b1a393adc56d0be6cb1b45c668a8ba413bf91a59a8b1eb76633f0f8e3429ff15c6a0f03a9404e3e4f0d3bacd6f2cbd0ef3984ac5b0e0ad7
-  data.tar.gz: 63f911fce5ddf75e7565095c134ffa0a3808fc5bf2b8e86f62466092b32a127afa449cbb44007774c771838fd30222a957246cbb44e9b8b125b01c2a7e8f2fe0
+  metadata.gz: 86e29718ae2772221fe62c2ce6e482cbecdb18cefe320e8f0cf408f6bb6826e1b90ea78c704e72818016e77c4f6b8537620043df2f00532d69c2ba505a2c52c7
+  data.tar.gz: 30d839adc80bdcb4043644d162a7a368080a0eaf3bc6fb82c26b994e61e0580991a6b2622188fa8dd6c07cdf78b3198deadb8cc22533743400fecd38dcbc392f

data/db/migrate/20101216224513_create_delayed_jobs.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class CreateDelayedJobs < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Backend::ActiveRecord::Job.connection

data/db/migrate/20110208031356_add_delayed_jobs_tag.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class AddDelayedJobsTag < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Backend::ActiveRecord::Job.connection

data/db/migrate/20110426161613_add_delayed_jobs_max_attempts.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class AddDelayedJobsMaxAttempts < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Backend::ActiveRecord::Job.connection

data/db/migrate/20110516225834_add_delayed_jobs_strand.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class AddDelayedJobsStrand < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Backend::ActiveRecord::Job.connection

data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class CleanupDelayedJobsIndexes < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Backend::ActiveRecord::Job.connection

data/db/migrate/20110610213249_optimize_delayed_jobs.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class OptimizeDelayedJobs < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Backend::ActiveRecord::Job.connection

data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class AddDelayedJobsNextInStrand < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Backend::ActiveRecord::Job.connection

data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class DelayedJobsDeleteTriggerLockForUpdate < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Backend::ActiveRecord::Job.connection

data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class DropPsqlJobsPopFn < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Backend::ActiveRecord::Job.connection

data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class DelayedJobsUseAdvisoryLocks < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Backend::ActiveRecord::Job.connection

data/db/migrate/20120607181141_index_jobs_on_locked_by.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class IndexJobsOnLockedBy < ActiveRecord::Migration[4.2]
   disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
 

data/db/migrate/20120608191051_add_jobs_run_at_index.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class AddJobsRunAtIndex < ActiveRecord::Migration[4.2]
   disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
 

data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ChangeDelayedJobsHandlerToText < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Job.connection

data/db/migrate/20140505215131_add_failed_jobs_original_job_id.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class AddFailedJobsOriginalJobId < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Backend::ActiveRecord::Job.connection

data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class CopyFailedJobsOriginalId < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Backend::ActiveRecord::Job.connection

data/db/migrate/20140505223637_drop_failed_jobs_original_id.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class DropFailedJobsOriginalId < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Backend::ActiveRecord::Job.connection

data/db/migrate/20140512213941_add_source_to_jobs.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class AddSourceToJobs < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Job.connection

data/db/migrate/20150807133223_add_max_concurrent_to_jobs.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class AddMaxConcurrentToJobs < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Job.connection

data/db/migrate/20151123210429_add_expires_at_to_jobs.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class AddExpiresAtToJobs < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Job.connection

data/db/migrate/20151210162949_improve_max_concurrent.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ImproveMaxConcurrent < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Job.connection

data/db/migrate/20161206323555_add_back_default_string_limits_jobs.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class AddBackDefaultStringLimitsJobs < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Job.connection

data/db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class SpeedUpMaxConcurrentTriggers  < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Job.connection

data/db/migrate/20190726154743_make_critical_columns_not_null.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class MakeCriticalColumnsNotNull < ActiveRecord::Migration[4.2]
   def connection
     Delayed::Job.connection

data/db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class AddIdToGetDelayedJobsIndex < ActiveRecord::Migration[4.2]
   disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
 

data/db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+class SpeedUpMaxConcurrentDeleteTrigger < ActiveRecord::Migration[4.2]
+  def connection
+    Delayed::Job.connection
+  end
+
+  def up
+    if connection.adapter_name == 'PostgreSQL'
+      # tl;dr sacrifice some responsiveness to max_concurrent changes for faster performance
+      # don't get the count every single time - it's usually safe to just set the next one in line
+      # since the max_concurrent doesn't change all that often for a strand
+      execute(<<-SQL)
+        CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+        DECLARE
+          running_count integer;
+          should_lock boolean;
+          should_be_precise boolean;
+        BEGIN
+          IF OLD.strand IS NOT NULL THEN
+            should_lock := true;
+            should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+        
+            IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+              running_count := (SELECT COUNT(*) FROM (
+                SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+              ) subquery_for_count);
+              should_lock := running_count < OLD.max_concurrent;
+            END IF;
+        
+            IF should_lock THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+            END IF;
+        
+            IF should_be_precise THEN
+              running_count := (SELECT COUNT(*) FROM (
+                SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+              ) subquery_for_count);
+              IF running_count < OLD.max_concurrent THEN
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
+                  SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                  j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
+                );
+              END IF;
+            ELSE
+              -- n-strands don't require precise ordering; we can make this query more performant
+              IF OLD.max_concurrent > 1 THEN
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                  j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
+              ELSE
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                  (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                    j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
+              END IF;
+            END IF;
+          END IF;
+          RETURN OLD;
+        END;
+        $$ LANGUAGE plpgsql;
+      SQL
+    end
+  end
+
+  def down
+    if connection.adapter_name == 'PostgreSQL'
+      execute(<<-SQL)
+        CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+        DECLARE
+          running_count integer;
+        BEGIN
+          IF OLD.strand IS NOT NULL THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+            IF OLD.id % 20 = 0 THEN
+              running_count := (SELECT COUNT(*) FROM (
+                SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+              ) subquery_for_count);
+              IF running_count < OLD.max_concurrent THEN
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
+                  SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                  j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
+                );
+              END IF;
+            ELSE
+              UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                  j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
+            END IF;
+          END IF;
+          RETURN OLD;
+        END;
+        $$ LANGUAGE plpgsql;
+      SQL
+    end
+  end
+end
+

data/db/migrate/20200825011002_add_strand_order_override.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+class AddStrandOrderOverride < ActiveRecord::Migration[4.2]
+  disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
+
+  def connection
+    Delayed::Job.connection
+  end
+
+  def up
+    add_column :delayed_jobs, :strand_order_override, :integer, default: 0, null: false
+    add_column :failed_jobs, :strand_order_override, :integer, default: 0, null: false
+    add_index :delayed_jobs, [:strand, :strand_order_override, :id],
+      algorithm: :concurrently,
+      where: "strand IS NOT NULL",
+      name: "next_in_strand_index"
+
+    if connection.adapter_name == 'PostgreSQL'
+      # Use the strand_order_override as the primary sorting mechanism (useful when moving between jobs queues without preserving ID ordering)
+      execute(<<-SQL)
+        CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+        DECLARE
+          running_count integer;
+          should_lock boolean;
+          should_be_precise boolean;
+        BEGIN
+          IF OLD.strand IS NOT NULL THEN
+            should_lock := true;
+            should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+        
+            IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+              running_count := (SELECT COUNT(*) FROM (
+                SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+              ) subquery_for_count);
+              should_lock := running_count < OLD.max_concurrent;
+            END IF;
+        
+            IF should_lock THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+            END IF;
+        
+            IF should_be_precise THEN
+              running_count := (SELECT COUNT(*) FROM (
+                SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+              ) subquery_for_count);
+              IF running_count < OLD.max_concurrent THEN
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
+                  SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                  j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
+                );
+              END IF;
+            ELSE
+              -- n-strands don't require precise ordering; we can make this query more performant
+              IF OLD.max_concurrent > 1 THEN
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                  j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
+              ELSE
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                  (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                    j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE);
+              END IF;
+            END IF;
+          END IF;
+          RETURN OLD;
+        END;
+        $$ LANGUAGE plpgsql;
+      SQL
+    end
+  end
+
+  def down
+    remove_column :delayed_jobs, :strand_order_override, :integer
+    remove_column :failed_jobs, :strand_order_override, :integer
+
+    if connection.adapter_name == 'PostgreSQL'
+      execute(<<-SQL)
+        CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+        DECLARE
+          running_count integer;
+          should_lock boolean;
+          should_be_precise boolean;
+        BEGIN
+          IF OLD.strand IS NOT NULL THEN
+            should_lock := true;
+            should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+        
+            IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+              running_count := (SELECT COUNT(*) FROM (
+                SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+              ) subquery_for_count);
+              should_lock := running_count < OLD.max_concurrent;
+            END IF;
+        
+            IF should_lock THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+            END IF;
+        
+            IF should_be_precise THEN
+              running_count := (SELECT COUNT(*) FROM (
+                SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+              ) subquery_for_count);
+              IF running_count < OLD.max_concurrent THEN
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
+                  SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                  j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
+                );
+              END IF;
+            ELSE
+              -- n-strands don't require precise ordering; we can make this query more performant
+              IF OLD.max_concurrent > 1 THEN
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                  j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
+              ELSE
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                  (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                    j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
+              END IF;
+            END IF;
+          END IF;
+          RETURN OLD;
+        END;
+        $$ LANGUAGE plpgsql;
+      SQL
+    end
+  end
+end

data/lib/delayed/backend/active_record.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class ActiveRecord::Base
   def self.load_for_delayed_job(id)
     if id
@@ -17,6 +19,8 @@ module Delayed
         include Delayed::Backend::Base
         self.table_name = :delayed_jobs
 
+        scope :next_in_strand_order, -> { order(:strand_order_override, :id) }
+
         def self.reconnect!
           clear_all_connections!
         end
@@ -103,7 +107,7 @@ module Delayed
         # so rather than changing the strand and balancing at queue time,
         # this keeps the strand intact and uses triggers to limit the number running
         def self.n_strand_options(strand_name, num_strands)
-          {:strand => strand_name, :max_concurrent => num_strands}
+          { strand: strand_name, max_concurrent: num_strands }
         end
 
         def self.current
@@ -123,8 +127,8 @@ module Delayed
         end
 
         # a nice stress test:
-        # 10_000.times { |i| Kernel.send_later_enqueue_args(:system, { :strand => 's1', :run_at => (24.hours.ago + (rand(24.hours.to_i))) }, "echo #{i} >> test1.txt") }
-        # 500.times { |i| "ohai".send_later_enqueue_args(:reverse, { :run_at => (12.hours.ago + (rand(24.hours.to_i))) }) }
+        # 10_000.times { |i| Kernel.delay(strand: 's1', run_at: (24.hours.ago + (rand(24.hours.to_i))).system("echo #{i} >> test1.txt") }
+        # 500.times { |i| "ohai".delay(run_at: (12.hours.ago + (rand(24.hours.to_i))).reverse }
         # then fire up your workers
         # you can check out strand correctness: diff test1.txt <(sort -n test1.txt)
          def self.ready_to_run(forced_latency: nil)
@@ -286,7 +290,7 @@ module Delayed
                     lock(lock)
                 jobs_with_row_number = all.from(target_jobs).
                     select("id, ROW_NUMBER() OVER () AS row_number")
-                updates = "locked_by = CASE row_number "
+                updates = +"locked_by = CASE row_number "
                 effective_worker_names.each_with_index do |worker, i|
                   updates << "WHEN #{i + 1} THEN #{connection.quote(worker)} "
                 end
@@ -398,7 +402,7 @@ module Delayed
           strand = options[:strand]
           on_conflict = options.delete(:on_conflict) || :use_earliest
           transaction_for_singleton(strand, on_conflict) do
-            job = self.where(:strand => strand, :locked_at => nil).order(:id).first
+            job = self.where(:strand => strand, :locked_at => nil).next_in_strand_order.first
             new_job = new(options)
             if job
               new_job.initialize_defaults