inst-jobs 2.3.1 → 2.4.8

data/lib/delayed/backend/active_record.rb

@@ -1,11 +1,13 @@
 # frozen_string_literal: true
 
-class ActiveRecord::Base
-  def self.load_for_delayed_job(id)
-    if id
-      find(id)
-    else
-      super
+module ActiveRecord
+  class Base
+    def self.load_for_delayed_job(id)
+      if id
+        find(id)
+      else
+        super
+      end
     end
   end
 end
@@ -31,17 +33,25 @@ module Delayed
 
         class << self
           def create(attributes, &block)
-            return super if connection.prepared_statements
-
+            on_conflict = attributes.delete(:on_conflict)
             # modified from ActiveRecord::Persistence.create and ActiveRecord::Persistence#_insert_record
             job = new(attributes, &block)
-            job.single_step_create
+            job.single_step_create(on_conflict: on_conflict)
+          end
+
+          def attempt_advisory_lock(lock_name)
+            fn_name = connection.quote_table_name("half_md5_as_bigint")
+            connection.select_value("SELECT pg_try_advisory_xact_lock(#{fn_name}('#{lock_name}'));")
+          end
+
+          def advisory_lock(lock_name)
+            fn_name = connection.quote_table_name("half_md5_as_bigint")
+            connection.execute("SELECT pg_advisory_xact_lock(#{fn_name}('#{lock_name}'));")
           end
         end
 
-        def single_step_create
+        def single_step_create(on_conflict: nil)
           connection = self.class.connection
-          return save if connection.prepared_statements
 
           # a before_save callback that we're skipping
           initialize_defaults
@@ -49,12 +59,10 @@ module Delayed
           current_time = current_time_from_proper_timezone
 
           all_timestamp_attributes_in_model.each do |column|
-            if !attribute_present?(column)
-              _write_attribute(column, current_time)
-            end
+            _write_attribute(column, current_time) unless attribute_present?(column)
           end
 
-          if Rails.version >= '6'
+          if Rails.version >= "6"
             attribute_names = attribute_names_for_partial_writes
             attribute_names = attributes_for_create(attribute_names)
             values = attributes_with_values(attribute_names)
@@ -62,21 +70,57 @@ module Delayed
             attribute_names = partial_writes? ? keys_for_partial_write : self.attribute_names
             values = attributes_with_values_for_create(attribute_names)
           end
+
           im = self.class.arel_table.compile_insert(self.class.send(:_substitute_values, values))
-          sql, _binds = connection.send(:to_sql_and_binds, im, [])
+
+          lock_and_insert = values["strand"] && instance_of?(Job)
+          # can't use prepared statements if we're combining multiple statemenets
+          sql, binds = if lock_and_insert
+                         connection.unprepared_statement do
+                           connection.send(:to_sql_and_binds, im)
+                         end
+                       else
+                         connection.send(:to_sql_and_binds, im)
+                       end
+          sql = +sql
+
+          if singleton && instance_of?(Job)
+            sql << " ON CONFLICT (singleton) WHERE singleton IS NOT NULL AND locked_by IS NULL DO "
+            sql << case on_conflict
+                   when :patient, :loose
+                     "NOTHING"
+                   when :overwrite
+                     "UPDATE SET run_at=EXCLUDED.run_at, handler=EXCLUDED.handler"
+                   else # :use_earliest
+                     "UPDATE SET run_at=EXCLUDED.run_at WHERE EXCLUDED.run_at<delayed_jobs.run_at"
+                   end
+          end
 
           # https://www.postgresql.org/docs/9.5/libpq-exec.html
-          sql = "#{sql} RETURNING id"
-          # > Multiple queries sent in a single PQexec call are processed in a single transaction,
-          # unless there are explicit BEGIN/COMMIT commands included in the query string to divide
-          # it into multiple transactions.
-          # but we don't need to lock when inserting into Delayed::Failed
-          sql = "SELECT pg_advisory_xact_lock(#{connection.quote_table_name('half_md5_as_bigint')}(#{connection.quote(values['strand'])})); #{sql}" if values["strand"] && self.class == Job
-          result = connection.execute(sql, "#{self} Create")
-          self.id = result.values.first.first
-          result.clear
-          @new_record = false
-          changes_applied
+          sql << " RETURNING id"
+
+          if lock_and_insert
+            # > Multiple queries sent in a single PQexec call are processed in a single transaction,
+            # unless there are explicit BEGIN/COMMIT commands included in the query string to divide
+            # it into multiple transactions.
+            # but we don't need to lock when inserting into Delayed::Failed
+            if values["strand"] && instance_of?(Job)
+              fn_name = connection.quote_table_name("half_md5_as_bigint")
+              sql = "SELECT pg_advisory_xact_lock(#{fn_name}(#{connection.quote(values['strand'])})); #{sql}"
+            end
+            result = connection.execute(sql, "#{self.class} Create")
+            self.id = result.values.first&.first
+            result.clear
+          else
+            result = connection.exec_query(sql, "#{self.class} Create", binds)
+            self.id = connection.send(:last_inserted_id, result)
+          end
+
+          # it might not get set if there was an existing record, and we didn't update it
+          if id
+            @new_record = false
+            changes_applied
+          end
 
           self
         end
@@ -103,9 +147,11 @@ module Delayed
         # to raise the lock level
         before_create :lock_strand_on_create
         def lock_strand_on_create
-          if strand.present? && self.class == Job
-            self.class.connection.execute("SELECT pg_advisory_xact_lock(#{self.class.connection.quote_table_name('half_md5_as_bigint')}(#{self.class.connection.quote(strand)}))")
-          end
+          return unless strand.present? && instance_of?(Job)
+
+          fn_name = self.class.connection.quote_table_name("half_md5_as_bigint")
+          quoted_strand_name = self.class.connection.quote(strand)
+          self.class.connection.execute("SELECT pg_advisory_xact_lock(#{fn_name}(#{quoted_strand_name}))")
         end
 
         # This overwrites the previous behavior
@@ -124,7 +170,7 @@ module Delayed
         end
 
         def self.failed
-          where("failed_at IS NOT NULL")
+          where.not(failed_at: nil)
         end
 
         def self.running
@@ -132,51 +178,54 @@ module Delayed
         end
 
         # a nice stress test:
-        # 10_000.times { |i| Kernel.delay(strand: 's1', run_at: (24.hours.ago + (rand(24.hours.to_i))).system("echo #{i} >> test1.txt") }
+        # 10_000.times do |i|
+        #   Kernel.delay(strand: 's1', run_at: (24.hours.ago + (rand(24.hours.to_i))).system("echo #{i} >> test1.txt")
+        # end
         # 500.times { |i| "ohai".delay(run_at: (12.hours.ago + (rand(24.hours.to_i))).reverse }
         # then fire up your workers
         # you can check out strand correctness: diff test1.txt <(sort -n test1.txt)
-         def self.ready_to_run(forced_latency: nil)
-           now = db_time_now
-           now -= forced_latency if forced_latency
-           where("run_at<=? AND locked_at IS NULL AND next_in_strand=?", now, true)
-         end
+        def self.ready_to_run(forced_latency: nil)
+          now = db_time_now
+          now -= forced_latency if forced_latency
+          where("run_at<=? AND locked_at IS NULL AND next_in_strand=?", now, true)
+        end
+
         def self.by_priority
           order(:priority, :run_at, :id)
         end
 
         # When a worker is exiting, make sure we don't have any locked jobs.
         def self.clear_locks!(worker_name)
-          where(:locked_by => worker_name).update_all(:locked_by => nil, :locked_at => nil)
+          where(locked_by: worker_name).update_all(locked_by: nil, locked_at: nil)
         end
 
         def self.strand_size(strand)
-          self.where(:strand => strand).count
+          where(strand: strand).count
         end
 
-        def self.running_jobs()
-          self.running.order(:locked_at)
+        def self.running_jobs
+          running.order(:locked_at)
         end
 
         def self.scope_for_flavor(flavor, query)
           scope = case flavor.to_s
-          when 'current'
-            self.current
-          when 'future'
-            self.future
-          when 'failed'
-            Delayed::Job::Failed
-          when 'strand'
-            self.where(:strand => query)
-          when 'tag'
-            self.where(:tag => query)
-          else
-            raise ArgumentError, "invalid flavor: #{flavor.inspect}"
-          end
-
-          if %w(current future).include?(flavor.to_s)
+                  when "current"
+                    current
+                  when "future"
+                    future
+                  when "failed"
+                    Delayed::Job::Failed
+                  when "strand"
+                    where(strand: query)
+                  when "tag"
+                    where(tag: query)
+                  else
+                    raise ArgumentError, "invalid flavor: #{flavor.inspect}"
+                  end
+
+          if %w[current future].include?(flavor.to_s)
             queue = query.presence || Delayed::Settings.queue
-            scope = scope.where(:queue => queue)
+            scope = scope.where(queue: queue)
           end
 
           scope
@@ -193,8 +242,8 @@ module Delayed
                            limit,
                            offset = 0,
                            query = nil)
-          scope = self.scope_for_flavor(flavor, query)
-          order = flavor.to_s == 'future' ? 'run_at' : 'id desc'
+          scope = scope_for_flavor(flavor, query)
+          order = flavor.to_s == "future" ? "run_at" : "id desc"
           scope.order(order).limit(limit).offset(offset).to_a
         end
 
@@ -202,7 +251,7 @@ module Delayed
         # see list_jobs for documentation on arguments
         def self.jobs_count(flavor,
                             query = nil)
-          scope = self.scope_for_flavor(flavor, query)
+          scope = scope_for_flavor(flavor, query)
           scope.count
         end
 
@@ -211,30 +260,33 @@ module Delayed
         # to specify the jobs to act on, either pass opts[:ids] = [list of job ids]
         # or opts[:flavor] = <some flavor> to perform on all jobs of that flavor
         def self.bulk_update(action, opts)
-          raise("Can't #{action.to_s} failed jobs") if opts[:flavor].to_s == 'failed' && action.to_s != 'destroy'
+          raise("Can't #{action} failed jobs") if opts[:flavor].to_s == "failed" && action.to_s != "destroy"
+
           scope = if opts[:ids]
-            if opts[:flavor] == 'failed'
-              Delayed::Job::Failed.where(:id => opts[:ids])
-            else
-              self.where(:id => opts[:ids])
-            end
-          elsif opts[:flavor]
+                    if opts[:flavor] == "failed"
+                      Delayed::Job::Failed.where(id: opts[:ids])
+                    else
+                      where(id: opts[:ids])
+                    end
+                  elsif opts[:flavor]
 
-            self.scope_for_flavor(opts[:flavor], opts[:query])
-          end
+                    scope_for_flavor(opts[:flavor], opts[:query])
+                  end
 
           return 0 unless scope
 
           case action.to_s
-          when 'hold'
+          when "hold"
             scope = scope.where(locked_by: nil)
-            scope.update_all(:locked_by => ON_HOLD_LOCKED_BY, :locked_at => db_time_now, :attempts => ON_HOLD_COUNT)
-          when 'unhold'
+            scope.update_all(locked_by: ON_HOLD_LOCKED_BY, locked_at: db_time_now, attempts: ON_HOLD_COUNT)
+          when "unhold"
             now = db_time_now
             scope = scope.where(locked_by: ON_HOLD_LOCKED_BY)
-            scope.update_all(["locked_by = NULL, locked_at = NULL, attempts = 0, run_at = (CASE WHEN run_at > ? THEN run_at ELSE ? END), failed_at = NULL", now, now])
-          when 'destroy'
-            scope = scope.where("locked_by IS NULL OR locked_by=?", ON_HOLD_LOCKED_BY) unless opts[:flavor] == 'failed'
+            scope.update_all([<<~SQL.squish, now, now])
+              locked_by=NULL, locked_at=NULL, attempts=0, run_at=(CASE WHEN run_at > ? THEN run_at ELSE ? END), failed_at=NULL
+            SQL
+          when "destroy"
+            scope = scope.where("locked_by IS NULL OR locked_by=?", ON_HOLD_LOCKED_BY) unless opts[:flavor] == "failed"
             scope.delete_all
           end
         end
@@ -245,16 +297,17 @@ module Delayed
         def self.tag_counts(flavor,
                             limit,
                             offset = 0)
-          raise(ArgumentError, "invalid flavor: #{flavor}") unless %w(current all).include?(flavor.to_s)
+          raise(ArgumentError, "invalid flavor: #{flavor}") unless %w[current all].include?(flavor.to_s)
+
           scope = case flavor.to_s
-            when 'current'
-              self.current
-            when 'all'
-              self
-            end
+                  when "current"
+                    current
+                  when "all"
+                    self
+                  end
 
           scope = scope.group(:tag).offset(offset).limit(limit)
-          scope.order(Arel.sql("COUNT(tag) DESC")).count.map { |t,c| { :tag => t, :count => c } }
+          scope.order(Arel.sql("COUNT(tag) DESC")).count.map { |t, c| { tag: t, count: c } }
         end
 
         def self.maybe_silence_periodic_log(&block)
@@ -278,7 +331,7 @@ module Delayed
 
           loop do
             jobs = maybe_silence_periodic_log do
-              if connection.adapter_name == 'PostgreSQL' && !Settings.select_random_from_batch
+              if connection.adapter_name == "PostgreSQL" && !Settings.select_random_from_batch
                 # In Postgres, we can lock a job and return which row was locked in a single
                 # query by using RETURNING. Combine that with the ROW_NUMBER() window function
                 # to assign a distinct locked_at value to each job locked, when doing multiple
@@ -286,22 +339,20 @@ module Delayed
                 effective_worker_names = Array(worker_names)
 
                 lock = nil
-                lock = "FOR UPDATE SKIP LOCKED" if connection.postgresql_version >= 90500
+                lock = "FOR UPDATE SKIP LOCKED" if connection.postgresql_version >= 90_500
                 target_jobs = all_available(queue,
                                             min_priority,
                                             max_priority,
-                                            forced_latency: forced_latency).
-                    limit(effective_worker_names.length + prefetch).
-                    lock(lock)
-                jobs_with_row_number = all.from(target_jobs).
-                    select("id, ROW_NUMBER() OVER () AS row_number")
+                                            forced_latency: forced_latency)
+                              .limit(effective_worker_names.length + prefetch)
+                              .lock(lock)
+                jobs_with_row_number = all.from(target_jobs)
+                                          .select("id, ROW_NUMBER() OVER () AS row_number")
                 updates = +"locked_by = CASE row_number "
                 effective_worker_names.each_with_index do |worker, i|
                   updates << "WHEN #{i + 1} THEN #{connection.quote(worker)} "
                 end
-                if prefetch_owner
-                  updates << "ELSE #{connection.quote(prefetch_owner)} "
-                end
+                updates << "ELSE #{connection.quote(prefetch_owner)} " if prefetch_owner
                 updates << "END, locked_at = #{connection.quote(db_time_now)}"
 
                 # Originally this was done with a subquery, but this allows the query planner to
@@ -311,22 +362,22 @@ module Delayed
                 # For more details, see:
                 #  * https://dba.stackexchange.com/a/69497/55285
                 #  * https://github.com/feikesteenbergen/demos/blob/b7ecee8b2a79bf04cbcd74972e6bfb81903aee5d/bugs/update_limit_bug.txt
-                query = "WITH limited_jobs AS (#{jobs_with_row_number.to_sql}) " \
-                        "UPDATE #{quoted_table_name} SET #{updates} FROM limited_jobs WHERE limited_jobs.id=#{quoted_table_name}.id " \
-                        "RETURNING #{quoted_table_name}.*"
+                query = <<~SQL.squish
+                  WITH limited_jobs AS (#{jobs_with_row_number.to_sql})
+                  UPDATE #{quoted_table_name} SET #{updates} FROM limited_jobs WHERE limited_jobs.id=#{quoted_table_name}.id
+                  RETURNING #{quoted_table_name}.*
+                SQL
 
                 jobs = find_by_sql(query)
                 # because this is an atomic query, we don't have to return more jobs than we needed
                 # to try and lock them, nor is there a possibility we need to try again because
                 # all of the jobs we tried to lock had already been locked by someone else
-                if worker_names.is_a?(Array)
-                  result = jobs.index_by(&:locked_by)
-                  # all of the prefetched jobs can come back as an array
-                  result[prefetch_owner] = jobs.select { |j| j.locked_by == prefetch_owner } if prefetch_owner
-                  return result
-                else
-                  return jobs.first
-                end
+                return jobs.first unless worker_names.is_a?(Array)
+
+                result = jobs.index_by(&:locked_by)
+                # all of the prefetched jobs can come back as an array
+                result[prefetch_owner] = jobs.select { |j| j.locked_by == prefetch_owner } if prefetch_owner
+                return result
               else
                 batch_size = Settings.fetch_batch_size
                 batch_size *= worker_names.length if worker_names.is_a?(Array)
@@ -336,13 +387,13 @@ module Delayed
             if jobs.empty?
               return worker_names.is_a?(Array) ? {} : nil
             end
-            if Settings.select_random_from_batch
-              jobs = jobs.sort_by { rand }
-            end
+
+            jobs = jobs.sort_by { rand } if Settings.select_random_from_batch
             if worker_names.is_a?(Array)
               result = {}
               jobs.each do |job|
                 break if worker_names.empty?
+
                 worker_name = worker_names.first
                 if job.send(:lock_exclusively!, worker_name)
                   result[worker_name] = job
@@ -351,10 +402,10 @@ module Delayed
               end
               return result
             else
-              job = jobs.detect do |job|
+              locked_job = jobs.detect do |job|
                 job.send(:lock_exclusively!, worker_names)
               end
-              return job if job
+              return locked_job if locked_job
             end
           end
         end
@@ -376,27 +427,9 @@ module Delayed
           check_queue(queue)
           check_priorities(min_priority, max_priority)
 
-          self.ready_to_run(forced_latency: forced_latency).
-              where(:priority => min_priority..max_priority, :queue => queue).
-              by_priority
-        end
-
-        # used internally by create_singleton to take the appropriate lock
-        # depending on the db driver
-        def self.transaction_for_singleton(strand, on_conflict)
-          return yield if on_conflict == :loose
-          self.transaction do
-            if on_conflict == :patient
-              pg_function = 'pg_try_advisory_xact_lock'
-              execute_method = :select_value
-            else
-              pg_function = 'pg_advisory_xact_lock'
-              execute_method = :execute
-            end
-            result = connection.send(execute_method, sanitize_sql(["SELECT #{pg_function}(#{connection.quote_table_name('half_md5_as_bigint')}(?))", strand]))
-            return if result == false && on_conflict == :patient
-            yield
-          end
+          ready_to_run(forced_latency: forced_latency)
+            .where(priority: min_priority..max_priority, queue: queue)
+            .by_priority
         end
 
         # Create the job on the specified strand, but only if there aren't any
@@ -404,10 +437,11 @@ module Delayed
         # (in other words, the job will still be created if there's another job
         # on the strand but it's already running)
         def self.create_singleton(options)
-          strand = options[:strand]
+          strand = options[:singleton]
           on_conflict = options.delete(:on_conflict) || :use_earliest
-          transaction_for_singleton(strand, on_conflict) do
-            job = self.where(:strand => strand, :locked_at => nil).next_in_strand_order.first
+
+          transaction_for_singleton(singleton, on_conflict) do
+            job = where(strand: strand, locked_at: nil).next_in_strand_order.first
             new_job = new(options)
             if job
               new_job.initialize_defaults
@@ -431,12 +465,22 @@ module Delayed
 
         def self.processes_locked_locally(name: nil)
           name ||= Socket.gethostname rescue x
-          where("locked_by LIKE ?", "#{name}:%").pluck(:locked_by).map{|locked_by| locked_by.split(":").last.to_i}
+          where("locked_by LIKE ?", "#{name}:%").pluck(:locked_by).map { |locked_by| locked_by.split(":").last.to_i }
+        end
+
+        def self.prefetch_jobs_lock_name
+          "Delayed::Job.unlock_orphaned_prefetched_jobs"
         end
 
         def self.unlock_orphaned_prefetched_jobs
-          horizon = db_time_now - Settings.parent_process[:prefetched_jobs_timeout] * 4
-          where("locked_by LIKE 'prefetch:%' AND locked_at<?", horizon).update_all(locked_at: nil, locked_by: nil)
+          transaction do
+            # for db performance reasons, we only need one process doing this at a time
+            # so if we can't get an advisory lock, just abort. we'll try again soon
+            return unless attempt_advisory_lock(prefetch_jobs_lock_name)
+
+            horizon = db_time_now - (Settings.parent_process[:prefetched_jobs_timeout] * 4)
+            where("locked_by LIKE 'prefetch:%' AND locked_at<?", horizon).update_all(locked_at: nil, locked_by: nil)
+          end
         end
 
         def self.unlock(jobs)
@@ -454,12 +498,14 @@ module Delayed
         def lock_exclusively!(worker)
           now = self.class.db_time_now
           # We don't own this job so we will update the locked_by name and the locked_at
-          affected_rows = self.class.where("id=? AND locked_at IS NULL AND run_at<=?", self, now).update_all(:locked_at => now, :locked_by => worker)
+          affected_rows = self.class.where("id=? AND locked_at IS NULL AND run_at<=?", self, now).update_all(
+            locked_at: now, locked_by: worker
+          )
           if affected_rows == 1
             mark_as_locked!(now, worker)
-            return true
+            true
           else
-            return false
+            false
           end
         end
 
@@ -469,9 +515,9 @@ module Delayed
           affected_rows = self.class.where(id: self, locked_by: from).update_all(locked_at: now, locked_by: to)
           if affected_rows == 1
             mark_as_locked!(now, to)
-            return true
+            true
           else
-            return false
+            false
           end
         end
 
@@ -483,34 +529,43 @@ module Delayed
           if respond_to?(:changes_applied)
             changes_applied
           else
-            changed_attributes['locked_at'] = time
-            changed_attributes['locked_by'] = worker
+            changed_attributes["locked_at"] = time
+            changed_attributes["locked_by"] = worker
           end
         end
         protected :lock_exclusively!, :mark_as_locked!
 
         def create_and_lock!(worker)
           raise "job already exists" unless new_record?
+
+          # we don't want to process unique constraint violations of
+          # running singleton jobs; always do it as two steps
+          if singleton
+            single_step_create
+            lock_exclusively!(worker)
+            return
+          end
+
           self.locked_at = Delayed::Job.db_time_now
           self.locked_by = worker
           single_step_create
         end
 
         def fail!
-          attrs = self.attributes
-          attrs['original_job_id'] = attrs.delete('id')
-          attrs['failed_at'] ||= self.class.db_time_now
-          attrs.delete('next_in_strand')
-          attrs.delete('max_concurrent')
+          attrs = attributes
+          attrs["original_job_id"] = attrs.delete("id")
+          attrs["failed_at"] ||= self.class.db_time_now
+          attrs.delete("next_in_strand")
+          attrs.delete("max_concurrent")
           self.class.transaction do
             failed_job = Failed.create(attrs)
-            self.destroy
+            destroy
             failed_job
           end
         rescue
           # we got an error while failing the job -- we need to at least get
           # the job out of the queue
-          self.destroy
+          destroy
           # re-raise so the worker logs the error, at least
           raise
         end
@@ -520,7 +575,6 @@ module Delayed
           self.table_name = :failed_jobs
         end
       end
-
     end
   end
 end