inst-jobs 0.15.18 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1f5fa07aabb776db7b07347664cd263fae34426050ea23b72e2f35ef6a665f46
-  data.tar.gz: cfb860844d1a2c3654a8047f8c32331fab957f6fd1d5ac77f641f2704849d738
+  metadata.gz: d1547b62f3c5fd86703cc209ec4b0feb6954db3c6fb4b58f4bcdbcf58dc9a9eb
+  data.tar.gz: 95ad1cacd71ac4511ee29eb0d3fda780037768234c232302b6fc10084de9e351
 SHA512:
-  metadata.gz: 9dd503fd8af0128fc999a78f96b199f759ea1c9df121f816ee72dd288d2e6bc6480b69bdece15f8f32c167e9785d18d9fb4b262abfd8459e08c12771d3dbe606
-  data.tar.gz: cb4f63db74b2cd05fde625c43e100c07cd66ded4f3b0998d97bb19786befde179180b85b1dd45d193254bdd010aba9e2ddaf67b549d0e8cdc0657853b99146ed
+  metadata.gz: bd27fc7e6c0ca5a613ba98ea0928d7947f2344d05c57ab14f588ff615c69bab60a0b6f08e60d1b610b6746ad8a467d151237f1469129f6144d5e93ce20ea71ef
+  data.tar.gz: 25ce44b9f6eeedae9605c4bc2077a9e9b51a4b7df94ba4d68be1e78f70801af65273c332caba780c11bf523cff31daed9c44d8f068f697a8910781b09a2daf88

data/db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb

@@ -0,0 +1,95 @@
+class SpeedUpMaxConcurrentDeleteTrigger < ActiveRecord::Migration[4.2]
+  def connection
+    Delayed::Job.connection
+  end
+
+  def up
+    if connection.adapter_name == 'PostgreSQL'
+      # tl;dr sacrifice some responsiveness to max_concurrent changes for faster performance
+      # don't get the count every single time - it's usually safe to just set the next one in line
+      # since the max_concurrent doesn't change all that often for a strand
+      execute(<<-SQL)
+        CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+        DECLARE
+          running_count integer;
+          should_lock boolean;
+          should_be_precise boolean;
+        BEGIN
+          IF OLD.strand IS NOT NULL THEN
+            should_lock := true;
+            should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+        
+            IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+              running_count := (SELECT COUNT(*) FROM (
+                SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+              ) subquery_for_count);
+              should_lock := running_count < OLD.max_concurrent;
+            END IF;
+        
+            IF should_lock THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+            END IF;
+        
+            IF should_be_precise THEN
+              running_count := (SELECT COUNT(*) FROM (
+                SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+              ) subquery_for_count);
+              IF running_count < OLD.max_concurrent THEN
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
+                  SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                  j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
+                );
+              END IF;
+            ELSE
+              -- n-strands don't require precise ordering; we can make this query more performant
+              IF OLD.max_concurrent > 1 THEN
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                  j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
+              ELSE
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                  (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                    j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
+              END IF;
+            END IF;
+          END IF;
+          RETURN OLD;
+        END;
+        $$ LANGUAGE plpgsql;
+      SQL
+    end
+  end
+
+  def down
+    if connection.adapter_name == 'PostgreSQL'
+      execute(<<-SQL)
+        CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+        DECLARE
+          running_count integer;
+        BEGIN
+          IF OLD.strand IS NOT NULL THEN
+            PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+            IF OLD.id % 20 = 0 THEN
+              running_count := (SELECT COUNT(*) FROM (
+                SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+              ) subquery_for_count);
+              IF running_count < OLD.max_concurrent THEN
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
+                  SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                  j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
+                );
+              END IF;
+            ELSE
+              UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                  j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
+            END IF;
+          END IF;
+          RETURN OLD;
+        END;
+        $$ LANGUAGE plpgsql;
+      SQL
+    end
+  end
+end
+

data/db/migrate/20200825011002_add_strand_order_override.rb

@@ -0,0 +1,126 @@
+class AddStrandOrderOverride < ActiveRecord::Migration[4.2]
+  disable_ddl_transaction! if respond_to?(:disable_ddl_transaction!)
+
+  def connection
+    Delayed::Job.connection
+  end
+
+  def up
+    add_column :delayed_jobs, :strand_order_override, :integer, default: 0, null: false
+    add_column :failed_jobs, :strand_order_override, :integer, default: 0, null: false
+    add_index :delayed_jobs, [:strand, :strand_order_override, :id],
+      algorithm: :concurrently,
+      where: "strand IS NOT NULL",
+      name: "next_in_strand_index"
+
+    if connection.adapter_name == 'PostgreSQL'
+      # Use the strand_order_override as the primary sorting mechanism (useful when moving between jobs queues without preserving ID ordering)
+      execute(<<-SQL)
+        CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+        DECLARE
+          running_count integer;
+          should_lock boolean;
+          should_be_precise boolean;
+        BEGIN
+          IF OLD.strand IS NOT NULL THEN
+            should_lock := true;
+            should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+        
+            IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+              running_count := (SELECT COUNT(*) FROM (
+                SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+              ) subquery_for_count);
+              should_lock := running_count < OLD.max_concurrent;
+            END IF;
+        
+            IF should_lock THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+            END IF;
+        
+            IF should_be_precise THEN
+              running_count := (SELECT COUNT(*) FROM (
+                SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+              ) subquery_for_count);
+              IF running_count < OLD.max_concurrent THEN
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
+                  SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                  j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
+                );
+              END IF;
+            ELSE
+              -- n-strands don't require precise ordering; we can make this query more performant
+              IF OLD.max_concurrent > 1 THEN
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                  j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
+              ELSE
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                  (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                    j2.strand = OLD.strand ORDER BY j2.strand_order_override ASC, j2.id ASC LIMIT 1 FOR UPDATE);
+              END IF;
+            END IF;
+          END IF;
+          RETURN OLD;
+        END;
+        $$ LANGUAGE plpgsql;
+      SQL
+    end
+  end
+
+  def down
+    remove_column :delayed_jobs, :strand_order_override, :integer
+    remove_column :failed_jobs, :strand_order_override, :integer
+
+    if connection.adapter_name == 'PostgreSQL'
+      execute(<<-SQL)
+        CREATE OR REPLACE FUNCTION delayed_jobs_after_delete_row_tr_fn () RETURNS trigger AS $$
+        DECLARE
+          running_count integer;
+          should_lock boolean;
+          should_be_precise boolean;
+        BEGIN
+          IF OLD.strand IS NOT NULL THEN
+            should_lock := true;
+            should_be_precise := OLD.id % (OLD.max_concurrent * 4) = 0;
+        
+            IF NOT should_be_precise AND OLD.max_concurrent > 16 THEN
+              running_count := (SELECT COUNT(*) FROM (
+                SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+              ) subquery_for_count);
+              should_lock := running_count < OLD.max_concurrent;
+            END IF;
+        
+            IF should_lock THEN
+              PERFORM pg_advisory_xact_lock(half_md5_as_bigint(OLD.strand));
+            END IF;
+        
+            IF should_be_precise THEN
+              running_count := (SELECT COUNT(*) FROM (
+                SELECT 1 as one FROM delayed_jobs WHERE strand = OLD.strand AND next_in_strand = 't' LIMIT OLD.max_concurrent
+              ) subquery_for_count);
+              IF running_count < OLD.max_concurrent THEN
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id IN (
+                  SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                  j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT (OLD.max_concurrent - running_count) FOR UPDATE
+                );
+              END IF;
+            ELSE
+              -- n-strands don't require precise ordering; we can make this query more performant
+              IF OLD.max_concurrent > 1 THEN
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                  j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE SKIP LOCKED);
+              ELSE
+                UPDATE delayed_jobs SET next_in_strand = 't' WHERE id =
+                  (SELECT id FROM delayed_jobs j2 WHERE next_in_strand = 'f' AND
+                    j2.strand = OLD.strand ORDER BY j2.id ASC LIMIT 1 FOR UPDATE);
+              END IF;
+            END IF;
+          END IF;
+          RETURN OLD;
+        END;
+        $$ LANGUAGE plpgsql;
+      SQL
+    end
+  end
+end

data/lib/delayed/backend/active_record.rb

@@ -17,6 +17,8 @@ module Delayed
         include Delayed::Backend::Base
         self.table_name = :delayed_jobs
 
+        scope :next_in_strand_order, -> { order(:strand_order_override, :id) }
+
         def self.reconnect!
           clear_all_connections!
         end
@@ -27,42 +29,54 @@ module Delayed
 
             # modified from ActiveRecord::Persistence.create and ActiveRecord::Persistence#_insert_record
             job = new(attributes, &block)
-            # a before_save callback that we're skipping
-            job.initialize_defaults
+            job.single_step_create
+          end
+        end
 
-            current_time = current_time_from_proper_timezone
+        def single_step_create
+          connection = self.class.connection
+          return save if connection.prepared_statements || Rails.version < '5.2'
 
-            job.send(:all_timestamp_attributes_in_model).each do |column|
-              if !job.attribute_present?(column)
-                job._write_attribute(column, current_time)
-              end
-            end
+          # a before_save callback that we're skipping
+          initialize_defaults
 
-            if Rails.version >= '6'
-              attribute_names = job.send(:attribute_names_for_partial_writes)
-              attribute_names = job.send(:attributes_for_create, attribute_names)
-              values = job.send(:attributes_with_values, attribute_names)
-            else
-              attribute_names = job.partial_writes? ? job.send(:keys_for_partial_write) : self.attribute_names
-              values = job.send(:attributes_with_values_for_create, attribute_names)
+          current_time = current_time_from_proper_timezone
+
+          all_timestamp_attributes_in_model.each do |column|
+            if !attribute_present?(column)
+              _write_attribute(column, current_time)
             end
-            im = arel_table.compile_insert(_substitute_values(values))
-            sql, _binds = connection.send(:to_sql_and_binds, im, [])
-
-            # https://www.postgresql.org/docs/9.5/libpq-exec.html
-            sql = "#{sql} RETURNING id"
-            # > Multiple queries sent in a single PQexec call are processed in a single transaction,
-            # unless there are explicit BEGIN/COMMIT commands included in the query string to divide
-            # it into multiple transactions.
-            sql = "SELECT pg_advisory_xact_lock(#{connection.quote_table_name('half_md5_as_bigint')}(#{connection.quote(values['strand'])})); #{sql}" if values["strand"]
-            result = connection.execute(sql, "#{self} Create")
-            job.id = result.values.first.first
-            result.clear
-            job.instance_variable_set(:@new_record, false)
-            job.changes_applied
-
-            job
           end
+
+          if Rails.version >= '6'
+            attribute_names = attribute_names_for_partial_writes
+            attribute_names = attributes_for_create(attribute_names)
+            values = attributes_with_values(attribute_names)
+          else
+            attribute_names = partial_writes? ? keys_for_partial_write : self.attribute_names
+            values = attributes_with_values_for_create(attribute_names)
+          end
+          im = self.class.arel_table.compile_insert(self.class.send(:_substitute_values, values))
+          sql, _binds = connection.send(:to_sql_and_binds, im, [])
+
+          # https://www.postgresql.org/docs/9.5/libpq-exec.html
+          sql = "#{sql} RETURNING id"
+          # > Multiple queries sent in a single PQexec call are processed in a single transaction,
+          # unless there are explicit BEGIN/COMMIT commands included in the query string to divide
+          # it into multiple transactions.
+          sql = "SELECT pg_advisory_xact_lock(#{connection.quote_table_name('half_md5_as_bigint')}(#{connection.quote(values['strand'])})); #{sql}" if values["strand"]
+          result = connection.execute(sql, "#{self} Create")
+          self.id = result.values.first.first
+          result.clear
+          @new_record = false
+          changes_applied
+
+          self
+        end
+
+        def destroy
+          # skip transaction and callbacks
+          destroy_row
         end
 
         # be aware that some strand functionality is controlled by triggers on
@@ -365,7 +379,15 @@ module Delayed
         def self.transaction_for_singleton(strand, on_conflict)
           return yield if on_conflict == :loose
           self.transaction do
-            connection.execute(sanitize_sql(["SELECT pg_advisory_xact_lock(#{connection.quote_table_name('half_md5_as_bigint')}(?))", strand]))
+            if on_conflict == :patient
+              pg_function = 'pg_try_advisory_xact_lock'
+              execute_method = :select_value
+            else
+              pg_function = 'pg_advisory_xact_lock'
+              execute_method = :execute
+            end
+            result = connection.send(execute_method, sanitize_sql(["SELECT #{pg_function}(#{connection.quote_table_name('half_md5_as_bigint')}(?))", strand]))
+            return if result == false && on_conflict == :patient
             yield
           end
         end
@@ -378,13 +400,13 @@ module Delayed
           strand = options[:strand]
           on_conflict = options.delete(:on_conflict) || :use_earliest
           transaction_for_singleton(strand, on_conflict) do
-            job = self.where(:strand => strand, :locked_at => nil).order(:id).first
+            job = self.where(:strand => strand, :locked_at => nil).next_in_strand_order.first
             new_job = new(options)
             if job
               new_job.initialize_defaults
               job.run_at =
                 case on_conflict
-                when :use_earliest
+                when :use_earliest, :patient
                   [job.run_at, new_job.run_at].min
                 when :overwrite
                   new_job.run_at
@@ -464,7 +486,7 @@ module Delayed
           raise "job already exists" unless new_record?
           self.locked_at = Delayed::Job.db_time_now
           self.locked_by = worker
-          save!
+          single_step_create
         end
 
         def fail!

data/lib/delayed/periodic.rb

@@ -47,7 +47,10 @@ class Periodic
   end
 
   def enqueue
-    Delayed::Job.enqueue(self, @job_args.merge(:max_attempts => 1, :run_at => @cron.next_time(Delayed::Periodic.now).utc.to_time, :singleton => tag))
+    Delayed::Job.enqueue(self, @job_args.merge(:max_attempts => 1,
+                                               :run_at => @cron.next_time(Delayed::Periodic.now).utc.to_time,
+                                               :singleton => tag,
+                                               on_conflict: :patient))
   end
 
   def perform

data/lib/delayed/settings.rb

@@ -23,7 +23,10 @@ module Delayed
       :worker_health_check_config,
       :worker_procname_prefix,
     ]
-    SETTINGS_WITH_ARGS = [ :num_strands ]
+    SETTINGS_WITH_ARGS = [
+      :job_detailed_log_format,
+      :num_strands
+    ]
 
     SETTINGS.each do |setting|
       mattr_writer(setting)
@@ -65,6 +68,7 @@ module Delayed
 
     self.num_strands = ->(strand_name){ nil }
     self.default_job_options = ->{ Hash.new }
+    self.job_detailed_log_format = ->(job){ job.to_json(include_root: false, only: %w(tag strand priority attempts created_at max_attempts source)) }
 
     # Send workers KILL after QUIT if they haven't exited within the
     # slow_exit_timeout

data/lib/delayed/version.rb

@@ -1,3 +1,3 @@
 module Delayed
-  VERSION = "0.15.18"
+  VERSION = "0.16.0"
 end

data/lib/delayed/work_queue/parent_process/server.rb

@@ -5,7 +5,7 @@ class ParentProcess
     attr_reader :clients, :listen_socket
 
     include Delayed::Logging
-    SIGNALS = %i{INT TERM QUIT CHLD}
+    SIGNALS = %i{INT TERM QUIT}
 
     def initialize(listen_socket, parent_pid: nil, config: Settings.parent_process)
       @listen_socket = listen_socket

data/lib/delayed/worker.rb

@@ -253,7 +253,7 @@ class Worker
   def log_job(job, format = :short)
     case format
     when :long
-      "#{job.full_name} #{ job.to_json(:include_root => false, :only => %w(tag strand priority attempts created_at max_attempts source)) }"
+      "#{job.full_name} #{ Settings.job_detailed_log_format.call(job) }"
     else
       job.full_name
     end

data/spec/delayed/worker_spec.rb

@@ -38,6 +38,33 @@ describe Delayed::Worker do
     end
   end
 
+  describe "#log_job" do
+    around(:each) do |block|
+      prev_logger = Delayed::Settings.job_detailed_log_format
+      block.call
+      Delayed::Settings.job_detailed_log_format = prev_logger
+    end
+
+    it "has a reasonable default format" do
+      payload = double(perform: nil)
+      job = Delayed::Job.new(payload_object: payload, priority: 25, strand: "test_jobs")
+      short_log_format = subject.log_job(job, :short)
+      expect(short_log_format).to eq("RSpec::Mocks::Double")
+      long_format = subject.log_job(job, :long)
+      expect(long_format).to eq("RSpec::Mocks::Double {\"priority\":25,\"attempts\":0,\"created_at\":null,\"tag\":\"RSpec::Mocks::Double#perform\",\"max_attempts\":null,\"strand\":\"test_jobs\",\"source\":null}")
+    end
+
+    it "logging format can be changed with settings" do
+      Delayed::Settings.job_detailed_log_format = ->(job){ "override format #{job.strand}"}
+      payload = double(perform: nil)
+      job = Delayed::Job.new(payload_object: payload, priority: 25, strand: "test_jobs")
+      short_log_format = subject.log_job(job, :short)
+      expect(short_log_format).to eq("RSpec::Mocks::Double")
+      long_format = subject.log_job(job, :long)
+      expect(long_format).to eq("RSpec::Mocks::Double override format test_jobs")
+    end
+  end
+
   describe "#run" do
     it "passes extra config options through to the WorkQueue" do
       expect(subject.work_queue).to receive(:get_and_lock_next_available).