que 0.11.3 → 2.2.0

data/lib/que/connection_pool.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# A wrapper around whatever connection pool we're using. Mainly just asserts
+# that the source connection pool is reentrant and thread-safe.
+
+module Que
+  class ConnectionPool
+    def initialize(&block)
+      @connection_proc = block
+      @checked_out     = Set.new
+      @mutex           = Mutex.new
+      @thread_key      = "que_connection_pool_#{object_id}".to_sym
+    end
+
+    def checkout
+      # Do some asserting to ensure that the connection pool we're using is
+      # behaving properly.
+      @connection_proc.call do |conn|
+        # Did this pool already have a connection for this thread?
+        preexisting = wrapped = current_connection
+
+        begin
+          if preexisting
+            # If so, check that the connection we just got is the one we expect.
+            if preexisting.wrapped_connection.backend_pid != conn.backend_pid
+              raise Error, "Connection pool is not reentrant! previous: #{preexisting.wrapped_connection.inspect} now: #{conn.inspect}"
+            end
+          else
+            # If not, make sure that it wasn't promised to any other threads.
+            sync do
+              Que.assert(@checked_out.add?(conn.backend_pid)) do
+                "Connection pool didn't synchronize access properly! (entrance: #{conn.backend_pid})"
+              end
+            end
+
+            self.current_connection = wrapped = Connection.wrap(conn)
+          end
+
+          yield(wrapped)
+        ensure
+          if preexisting.nil?
+            # We're at the top level (about to return this connection to the
+            # pool we got it from), so mark it as no longer ours.
+            self.current_connection = nil
+
+            sync do
+              Que.assert(@checked_out.delete?(conn.backend_pid)) do
+                "Connection pool didn't synchronize access properly! (exit: #{conn.backend_pid})"
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def execute(*args)
+      checkout { |conn| conn.execute(*args) }
+    end
+
+    def in_transaction?
+      checkout { |conn| conn.in_transaction? }
+    end
+
+    private
+
+    def sync(&block)
+      @mutex.synchronize(&block)
+    end
+
+    def current_connection
+      Thread.current[@thread_key]
+    end
+
+    def current_connection=(c)
+      Thread.current[@thread_key] = c
+    end
+  end
+end

data/lib/que/job.rb

@@ -1,11 +1,58 @@
 # frozen_string_literal: true
 
+# The class that jobs should generally inherit from.
+
 module Que
   class Job
-    attr_reader :attrs
+    include JobMethods
+
+    MAXIMUM_TAGS_COUNT = 5
+    MAXIMUM_TAG_LENGTH = 100
+
+    SQL[:insert_job] =
+      %{
+        INSERT INTO public.que_jobs
+        (queue, priority, run_at, job_class, args, kwargs, data, job_schema_version)
+        VALUES
+        (
+          coalesce($1, 'default')::text,
+          coalesce($2, 100)::smallint,
+          coalesce($3, now())::timestamptz,
+          $4::text,
+          coalesce($5, '[]')::jsonb,
+          coalesce($6, '{}')::jsonb,
+          coalesce($7, '{}')::jsonb,
+          #{Que.job_schema_version}
+        )
+        RETURNING *
+      }
+
+    SQL[:bulk_insert_jobs] =
+      %{
+        WITH args_and_kwargs as (
+          SELECT * from json_to_recordset(coalesce($5, '[{args:{},kwargs:{}}]')::json) as x(args jsonb, kwargs jsonb)
+        )
+        INSERT INTO public.que_jobs
+        (queue, priority, run_at, job_class, args, kwargs, data, job_schema_version)
+        SELECT
+          coalesce($1, 'default')::text,
+          coalesce($2, 100)::smallint,
+          coalesce($3, now())::timestamptz,
+          $4::text,
+          args_and_kwargs.args,
+          args_and_kwargs.kwargs,
+          coalesce($6, '{}')::jsonb,
+          #{Que.job_schema_version}
+        FROM args_and_kwargs
+        RETURNING *
+      }
+
+    attr_reader :que_attrs
+    attr_accessor :que_error, :que_resolved
 
     def initialize(attrs)
-      @attrs = attrs
+      @que_attrs = attrs
+      Que.internal_log(:job_instantiate, self) { attrs }
     end
 
     # Subclasses should define their own run methods, but keep an empty one
@@ -13,139 +60,199 @@ module Que
     def run(*args)
     end
 
-    def _run
-      run(*attrs[:args])
-      destroy unless @destroyed
-    end
-
     private
 
-    def destroy
-      Que.execute :destroy_job, attrs.values_at(:queue, :priority, :run_at, :job_id)
-      @destroyed = true
+    # Have the job helper methods act on this object.
+    def que_target
+      self
     end
 
-    @retry_interval = proc { |count| count ** 4 + 3 }
-
     class << self
-      attr_reader :retry_interval
+      # Job class configuration options.
+      attr_accessor \
+        :run_synchronously,
+        :retry_interval,
+        :maximum_retry_count,
+        :queue,
+        :priority,
+        :run_at
 
       def enqueue(*args)
-        if args.last.is_a?(Hash)
-          options   = args.pop
-          queue     = options.delete(:queue) || '' if options.key?(:queue)
-          job_class = options.delete(:job_class)
-          run_at    = options.delete(:run_at)
-          priority  = options.delete(:priority)
-          args << options if options.any?
-        end
+        args, kwargs = Que.split_out_ruby2_keywords(args)
 
-        attrs = {:job_class => job_class || to_s, :args => args}
+        job_options = kwargs.delete(:job_options) || {}
 
-        warn "@default_run_at in #{to_s} has been deprecated and will be removed in Que version 1.0.0. Please use @run_at instead." if @default_run_at
+        if job_options[:tags]
+          if job_options[:tags].length > MAXIMUM_TAGS_COUNT
+            raise Que::Error, "Can't enqueue a job with more than #{MAXIMUM_TAGS_COUNT} tags! (passed #{job_options[:tags].length})"
+          end
 
-        if t = run_at || @run_at && @run_at.call || @default_run_at && @default_run_at.call
-          attrs[:run_at] = t
+          job_options[:tags].each do |tag|
+            if tag.length > MAXIMUM_TAG_LENGTH
+              raise Que::Error, "Can't enqueue a job with a tag longer than 100 characters! (\"#{tag}\")"
+            end
+          end
         end
 
-        warn "@default_priority in #{to_s} has been deprecated and will be removed in Que version 1.0.0. Please use @priority instead." if @default_priority
+        attrs = {
+          queue:    job_options[:queue]    || resolve_que_setting(:queue) || Que.default_queue,
+          priority: job_options[:priority] || resolve_que_setting(:priority),
+          run_at:   job_options[:run_at]   || resolve_que_setting(:run_at),
+          args:     args,
+          kwargs:   kwargs,
+          data:     job_options[:tags] ? { tags: job_options[:tags] } : {},
+          job_class: \
+            job_options[:job_class] || name ||
+              raise(Error, "Can't enqueue an anonymous subclass of Que::Job"),
+        }
 
-        if p = priority || @priority || @default_priority
-          attrs[:priority] = p
-        end
+        if Thread.current[:que_jobs_to_bulk_insert]
+          if self.name == 'ActiveJob::QueueAdapters::QueAdapter::JobWrapper'
+            raise Que::Error, "Que.bulk_enqueue does not support ActiveJob."
+          end
 
-        if q = queue || @queue
-          attrs[:queue] = q
-        end
+          raise Que::Error, "When using .bulk_enqueue, job_options must be passed to that method rather than .enqueue" unless job_options == {}
 
-        if Que.mode == :sync && !t
-          run(*attrs[:args])
+          Thread.current[:que_jobs_to_bulk_insert][:jobs_attrs] << attrs
+          new({})
+        elsif attrs[:run_at].nil? && resolve_que_setting(:run_synchronously)
+          attrs.merge!(
+            args: Que.deserialize_json(Que.serialize_json(attrs[:args])),
+            kwargs: Que.deserialize_json(Que.serialize_json(attrs[:kwargs])),
+            data: Que.deserialize_json(Que.serialize_json(attrs[:data])),
+          )
+          _run_attrs(attrs)
         else
-          values = Que.execute(:insert_job, attrs.values_at(:queue, :priority, :run_at, :job_class, :args)).first
-          Que.adapter.wake_worker_after_commit unless t
+          attrs.merge!(
+            args: Que.serialize_json(attrs[:args]),
+            kwargs: Que.serialize_json(attrs[:kwargs]),
+            data: Que.serialize_json(attrs[:data]),
+          )
+          values = Que.execute(
+            :insert_job,
+            attrs.values_at(:queue, :priority, :run_at, :job_class, :args, :kwargs, :data),
+          ).first
           new(values)
         end
       end
+      ruby2_keywords(:enqueue) if respond_to?(:ruby2_keywords, true)
 
-      def queue(*args)
-        warn "#{to_s}.queue(*args) is deprecated and will be removed in Que version 1.0.0. Please use #{to_s}.enqueue(*args) instead."
-        enqueue(*args)
+      def bulk_enqueue(job_options: {}, notify: false)
+        raise Que::Error, "Can't nest .bulk_enqueue" unless Thread.current[:que_jobs_to_bulk_insert].nil?
+        Thread.current[:que_jobs_to_bulk_insert] = { jobs_attrs: [], job_options: job_options }
+        yield
+        jobs_attrs = Thread.current[:que_jobs_to_bulk_insert][:jobs_attrs]
+        job_options = Thread.current[:que_jobs_to_bulk_insert][:job_options]
+        return [] if jobs_attrs.empty?
+        raise Que::Error, "When using .bulk_enqueue, all jobs enqueued must be of the same job class" unless jobs_attrs.map { |attrs| attrs[:job_class] }.uniq.one?
+        args_and_kwargs_array = jobs_attrs.map { |attrs| attrs.slice(:args, :kwargs) }
+        klass = job_options[:job_class] ? Que::Job : Que.constantize(jobs_attrs.first[:job_class])
+        klass._bulk_enqueue_insert(args_and_kwargs_array, job_options: job_options, notify: notify)
+      ensure
+        Thread.current[:que_jobs_to_bulk_insert] = nil
       end
 
-      def run(*args)
-        # Should not fail if there's no DB connection.
-        new(:args => args).tap { |job| job.run(*args) }
-      end
+      def _bulk_enqueue_insert(args_and_kwargs_array, job_options: {}, notify:)
+        raise 'Unexpected bulk args format' if !args_and_kwargs_array.is_a?(Array) || !args_and_kwargs_array.all? { |a| a.is_a?(Hash) }
 
-      def work(queue = '')
-        # Since we're taking session-level advisory locks, we have to hold the
-        # same connection throughout the process of getting a job, working it,
-        # deleting it, and removing the lock.
-        return_value =
-          Que.adapter.checkout do
-            begin
-              if job = Que.execute(:lock_job, [queue]).first
-                # Edge case: It's possible for the lock_job query to have
-                # grabbed a job that's already been worked, if it took its MVCC
-                # snapshot while the job was processing, but didn't attempt the
-                # advisory lock until it was finished. Since we have the lock, a
-                # previous worker would have deleted it by now, so we just
-                # double check that it still exists before working it.
-
-                # Note that there is currently no spec for this behavior, since
-                # I'm not sure how to reliably commit a transaction that deletes
-                # the job in a separate thread between lock_job and check_job.
-                if Que.execute(:check_job, job.values_at(:queue, :priority, :run_at, :job_id)).none?
-                  {:event => :job_race_condition}
-                else
-                  klass = class_for(job[:job_class])
-                  klass.new(job)._run
-                  {:event => :job_worked, :job => job}
-                end
-              else
-                {:event => :job_unavailable}
-              end
-            rescue => error
-              begin
-                if job
-                  count    = job[:error_count].to_i + 1
-                  interval = klass && klass.respond_to?(:retry_interval) && klass.retry_interval || retry_interval
-                  delay    = interval.respond_to?(:call) ? interval.call(count) : interval
-                  message  = "#{error.message}\n#{error.backtrace.join("\n")}"
-                  Que.execute :set_error, [count, delay, message] + job.values_at(:queue, :priority, :run_at, :job_id)
-                end
-              rescue
-                # If we can't reach the database for some reason, too bad, but
-                # don't let it crash the work loop.
-              end
-
-              if Que.error_handler
-                # Similarly, protect the work loop from a failure of the error handler.
-                Que.error_handler.call(error, job) rescue nil
-              end
-
-              return {:event => :job_errored, :error => error, :job => job}
-            ensure
-              # Clear the advisory lock we took when locking the job. Important
-              # to do this so that they don't pile up in the database. Again, if
-              # we can't reach the database, don't crash the work loop.
-              begin
-                Que.execute "SELECT pg_advisory_unlock($1)", [job[:job_id]] if job
-              rescue
-              end
+        if job_options[:tags]
+          if job_options[:tags].length > MAXIMUM_TAGS_COUNT
+            raise Que::Error, "Can't enqueue a job with more than #{MAXIMUM_TAGS_COUNT} tags! (passed #{job_options[:tags].length})"
+          end
+
+          job_options[:tags].each do |tag|
+            if tag.length > MAXIMUM_TAG_LENGTH
+              raise Que::Error, "Can't enqueue a job with a tag longer than 100 characters! (\"#{tag}\")"
             end
           end
+        end
+
+        args_and_kwargs_array = args_and_kwargs_array.map do |args_and_kwargs|
+          args_and_kwargs.merge(
+            args: args_and_kwargs.fetch(:args, []),
+            kwargs: args_and_kwargs.fetch(:kwargs, {}),
+          )
+        end
+
+        attrs = {
+          queue:    job_options[:queue]    || resolve_que_setting(:queue) || Que.default_queue,
+          priority: job_options[:priority] || resolve_que_setting(:priority),
+          run_at:   job_options[:run_at]   || resolve_que_setting(:run_at),
+          args_and_kwargs_array: args_and_kwargs_array,
+          data:     job_options[:tags] ? { tags: job_options[:tags] } : {},
+          job_class: \
+            job_options[:job_class] || name ||
+              raise(Error, "Can't enqueue an anonymous subclass of Que::Job"),
+        }
+
+        if attrs[:run_at].nil? && resolve_que_setting(:run_synchronously)
+          args_and_kwargs_array = Que.deserialize_json(Que.serialize_json(attrs.delete(:args_and_kwargs_array)))
+          args_and_kwargs_array.map do |args_and_kwargs|
+            _run_attrs(
+              attrs.merge(
+                args: args_and_kwargs.fetch(:args),
+                kwargs: args_and_kwargs.fetch(:kwargs),
+              ),
+            )
+          end
+        else
+          attrs.merge!(
+            args_and_kwargs_array: Que.serialize_json(attrs[:args_and_kwargs_array]),
+            data: Que.serialize_json(attrs[:data]),
+          )
+          values_array =
+            Que.transaction do
+              Que.execute('SET LOCAL que.skip_notify TO true') unless notify
+              Que.execute(
+                :bulk_insert_jobs,
+                attrs.values_at(:queue, :priority, :run_at, :job_class, :args_and_kwargs_array, :data),
+              )
+            end
+          values_array.map(&method(:new))
+        end
+      end
 
-        Que.adapter.cleanup!
+      def run(*args)
+        # Make sure things behave the same as they would have with a round-trip
+        # to the DB.
+        args, kwargs = Que.split_out_ruby2_keywords(args)
+        args = Que.deserialize_json(Que.serialize_json(args))
+        kwargs = Que.deserialize_json(Que.serialize_json(kwargs))
 
-        return_value
+        # Should not fail if there's no DB connection.
+        _run_attrs(args: args, kwargs: kwargs)
+      end
+      ruby2_keywords(:run) if respond_to?(:ruby2_keywords, true)
+
+      def resolve_que_setting(setting, *args)
+        value = send(setting) if respond_to?(setting)
+
+        if !value.nil?
+          value.respond_to?(:call) ? value.call(*args) : value
+        else
+          c = superclass
+          if c.respond_to?(:resolve_que_setting)
+            c.resolve_que_setting(setting, *args)
+          end
+        end
       end
 
       private
 
-      def class_for(string)
-        Que.constantize(string)
+      def _run_attrs(attrs)
+        attrs[:error_count] = 0
+        Que.recursively_freeze(attrs)
+
+        new(attrs).tap do |job|
+          Que.run_job_middleware(job) do
+            job._run(reraise_errors: true)
+          end
+        end
       end
     end
+
+    # Set up some defaults.
+    self.retry_interval      = proc { |count| count ** 4 + 3 }
+    self.maximum_retry_count = 15
   end
 end

data/lib/que/job_buffer.rb

@@ -0,0 +1,255 @@
+# frozen_string_literal: true
+
+# A sized thread-safe queue that holds ordered job sort_keys. Supports blocking
+# while waiting for a job to become available, only returning jobs over a
+# minimum priority, and stopping gracefully.
+
+module Que
+  class JobBuffer
+    attr_reader :maximum_size, :priority_queues
+
+    # Since we use a mutex, which is not reentrant, we have to be a little
+    # careful to not call a method that locks the mutex when we've already
+    # locked it. So, as a general rule, public methods handle locking the mutex
+    # when necessary, while private methods handle the actual underlying data
+    # changes. This lets us reuse those private methods without running into
+    # locking issues.
+
+    def initialize(
+      maximum_size:,
+      priorities:
+    )
+      @maximum_size = Que.assert(Integer, maximum_size)
+      Que.assert(maximum_size >= 0) { "maximum_size for a JobBuffer must be at least zero!" }
+
+      @stop  = false
+      @array = []
+      @mutex = Mutex.new
+
+      @priority_queues = Hash[
+        # Make sure that priority = nil sorts highest.
+        priorities.sort_by{|p| p || MAXIMUM_PRIORITY}.map do |p|
+          [p, PriorityQueue.new(priority: p, job_buffer: self)]
+        end
+      ].freeze
+    end
+
+    def push(*metajobs)
+      Que.internal_log(:job_buffer_push, self) do
+        {
+          maximum_size:  maximum_size,
+          ids:           metajobs.map(&:id),
+          current_queue: to_a,
+        }
+      end
+
+      sync do
+        return metajobs if _stopping?
+
+        @array.concat(metajobs).sort!
+
+        # Relying on the hash's contents being sorted, here.
+        priority_queues.reverse_each do |_, pq|
+          pq.populate do
+            _shift_job(pq.priority)
+          end
+        end
+
+        # If we passed the maximum buffer size, drop the lowest sort keys and
+        # return their ids to be unlocked.
+        overage = -_buffer_space
+        pop(overage) if overage > 0
+      end
+    end
+
+    def shift(priority = nil)
+      queue = priority_queues.fetch(priority) { raise Error, "not a permitted priority! #{priority}" }
+      queue.pop || shift_job(priority)
+    end
+
+    def shift_job(priority = nil)
+      sync { _shift_job(priority) }
+    end
+
+    def accept?(metajobs)
+      metajobs.sort!
+
+      sync do
+        return [] if _stopping?
+
+        start_index = _buffer_space
+        final_index = metajobs.length - 1
+
+        return metajobs if start_index > final_index
+        index_to_lose = @array.length - 1
+
+        start_index.upto(final_index) do |index|
+          if index_to_lose >= 0 && (metajobs[index] <=> @array[index_to_lose]) < 0
+            return metajobs if index == final_index
+            index_to_lose -= 1
+          else
+            return metajobs.slice(0...index)
+          end
+        end
+
+        []
+      end
+    end
+
+    def waiting_count
+      count = 0
+      priority_queues.each_value do |pq|
+        count += pq.waiting_count
+      end
+      count
+    end
+
+    def available_priorities
+      hash = {}
+      lowest_priority = true
+
+      priority_queues.reverse_each do |priority, pq|
+        count = pq.waiting_count
+
+        if lowest_priority
+          count += buffer_space
+          lowest_priority = false
+        end
+
+        hash[priority || MAXIMUM_PRIORITY] = count if count > 0
+      end
+
+      hash
+    end
+
+    def buffer_space
+      sync { _buffer_space }
+    end
+
+    def size
+      sync { _size }
+    end
+
+    def to_a
+      sync { @array.dup }
+    end
+
+    def stop
+      sync { @stop = true }
+      priority_queues.each_value(&:stop)
+    end
+
+    def clear
+      sync { pop(_size) }
+    end
+
+    def stopping?
+      sync { _stopping? }
+    end
+
+    def job_available?(priority)
+      (job = @array.first) && job.priority_sufficient?(priority)
+    end
+
+    private
+
+    def _buffer_space
+      maximum_size - _size
+    end
+
+    def pop(count)
+      @array.pop(count)
+    end
+
+    def _shift_job(priority)
+      if _stopping?
+        false
+      elsif (job = @array.first) && job.priority_sufficient?(priority)
+        @array.shift
+      end
+    end
+
+    def _size
+      @array.size
+    end
+
+    def _stopping?
+      !!@stop
+    end
+
+    def sync(&block)
+      @mutex.synchronize(&block)
+    end
+
+    # A queue object dedicated to a specific worker priority. It's basically a
+    # Queue object from the standard library, but it's able to reach into the
+    # JobBuffer's buffer in order to satisfy a pop.
+    class PriorityQueue
+      attr_reader :job_buffer, :priority, :mutex
+
+      def initialize(
+        job_buffer:,
+        priority:
+      )
+        @job_buffer = job_buffer
+        @priority   = priority
+        @waiting    = 0
+        @stopping   = false
+        @items      = [] # Items pending distribution to waiting threads.
+        @mutex      = Mutex.new
+        @cv         = ConditionVariable.new
+      end
+
+      def pop
+        sync do
+          loop do
+            if @stopping
+              return false
+            elsif item = @items.pop
+              return item
+            elsif job_buffer.job_available?(priority)
+              return false
+            end
+
+            @waiting += 1
+            @cv.wait(mutex)
+            @waiting -= 1
+          end
+        end
+      end
+
+      def stop
+        sync do
+          @stopping = true
+          @cv.broadcast
+        end
+      end
+
+      def populate
+        sync do
+          waiting_count.times do
+            job = yield
+            break if job.nil? # False would mean we're stopping.
+            _push(job)
+          end
+        end
+      end
+
+      def waiting_count
+        @waiting
+      end
+
+      private
+
+      def sync(&block)
+        mutex.synchronize(&block)
+      end
+
+      def _push(item)
+        Que.assert(waiting_count > 0)
+        @items << item
+        @cv.signal
+      end
+    end
+  end
+end