que 0.11.3 → 2.2.0

data/lib/que/utils/middleware.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# Logic for middleware to wrap jobs.
+
+module Que
+  module Utils
+    module Middleware
+      TYPES = [
+        :job,
+        :sql,
+      ].freeze
+
+      TYPES.each do |type|
+        module_eval <<-CODE
+          def #{type}_middleware
+            @#{type}_middleware ||= []
+          end
+
+          def run_#{type}_middleware(*args)
+            m = #{type}_middleware
+
+            if m.empty?
+              yield
+            else
+              invoke_middleware(middleware: m.dup, args: args) { yield }
+            end
+          end
+        CODE
+      end
+
+      private
+
+      def invoke_middleware(middleware:, args:, &block)
+        if m = middleware.shift
+          r = nil
+          m.call(*args) do
+            r = invoke_middleware(middleware: middleware, args: args, &block)
+          end
+          r
+        else
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/que/utils/queue_management.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Tools for managing the contents/state of the queue.
+
+module Que
+  module Utils
+    module QueueManagement
+      def clear!
+        execute "DELETE FROM que_jobs"
+      end
+
+      # Very old migrations may use Que.create! and Que.drop!, which just
+      # created and dropped the initial version of the jobs table.
+      def create!; migrate!(version: 1); end
+      def drop!;   migrate!(version: 0); end
+    end
+  end
+end

data/lib/que/utils/ruby2_keywords.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# Temporary module allowing ruby2 keyword args to be extracted from an *args splat
+# Allows us to ensure consistent behaviour when running on ruby 2 vs ruby 3
+# We can remove this if/when we drop support for ruby 2
+
+require 'json'
+
+module Que
+  module Utils
+    module Ruby2Keywords
+      def split_out_ruby2_keywords(args)
+        return [args, {}] unless args.last&.is_a?(Hash) && Hash.ruby2_keywords_hash?(args.last)
+
+        [args[0..-2], args.last]
+      end
+    end
+  end
+end

data/lib/que/utils/transactions.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# A helper method to manage transactions, used mainly by the migration system.
+# It's available for general use, but if you're using an ORM that provides its
+# own transaction helper, be sure to use that instead, or the two may interfere
+# with one another.
+
+module Que
+  module Utils
+    module Transactions
+      def transaction
+        pool.checkout do
+          if pool.in_transaction?
+            yield
+          else
+            begin
+              execute "BEGIN"
+              yield
+            rescue => error
+              raise
+            ensure
+              # Handle a raised error or a killed thread.
+              if error || Thread.current.status == 'aborting'
+                execute "ROLLBACK"
+              else
+                execute "COMMIT"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/que/version.rb

@@ -1,5 +1,9 @@
 # frozen_string_literal: true
 
 module Que
-  Version = '0.11.3'
+  VERSION = '2.2.0'
+
+  def self.job_schema_version
+    2
+  end
 end

data/lib/que/worker.rb

@@ -1,184 +1,180 @@
 # frozen_string_literal: true
 
-require 'monitor'
+# Workers wrap threads which continuously pull job pks from JobBuffer objects,
+# fetch and work those jobs, and export relevant data to ResultQueues.
+
+require 'set'
 
 module Que
   class Worker
-    # Each worker has a thread that does the actual work of running jobs.
-    # Since both the worker's thread and whatever thread is managing the
-    # worker are capable of affecting the worker's state, we need to
-    # synchronize access to it.
-    include MonitorMixin
-
-    attr_reader :thread, :state, :queue
-
-    def initialize(queue = '')
-      super() # For MonitorMixin.
-      @queue  = queue
-      @state  = :working
-      @thread = Thread.new { work_loop }
-      @thread.abort_on_exception = true
-    end
-
-    def alive?
-      !!@thread.status
-    end
-
-    def sleeping?
-      synchronize { _sleeping? }
-    end
-
-    def working?
-      synchronize { @state == :working }
-    end
-
-    def wake!
-      synchronize do
-        if sleeping?
-          # Have to set the state here so that another thread checking
-          # immediately after this won't see the worker as asleep.
-          @state = :working
-          @thread.wakeup
-          true
-        end
+    attr_reader :thread, :priority
+
+    VALID_LOG_LEVELS = [:debug, :info, :warn, :error, :fatal, :unknown].to_set.freeze
+
+    SQL[:check_job] =
+      %{
+        SELECT 1 AS one
+        FROM public.que_jobs
+        WHERE id = $1::bigint
+      }
+
+    def initialize(
+      job_buffer:,
+      result_queue:,
+      priority: nil,
+      start_callback: nil
+    )
+
+      @priority     = Que.assert([NilClass, Integer], priority)
+      @job_buffer   = Que.assert(JobBuffer, job_buffer)
+      @result_queue = Que.assert(ResultQueue, result_queue)
+
+      Que.internal_log(:worker_instantiate, self) do
+        {
+          priority:     priority,
+          job_buffer:   job_buffer.object_id,
+          result_queue: result_queue.object_id,
+        }
       end
-    end
 
-    # This needs to be called when trapping a signal, so it can't lock the monitor.
-    def stop
-      @stop = true
-      @thread.wakeup if _sleeping?
+      @thread =
+        Thread.new do
+          # An error causing this thread to exit is a bug in Que, which we want
+          # to know about ASAP, so propagate the error if it happens.
+          Thread.current.abort_on_exception = true
+          start_callback.call(self) if start_callback.respond_to?(:call)
+          work_loop
+        end
     end
 
     def wait_until_stopped
-      wait while alive?
+      @thread.join
     end
 
     private
 
-    # Sleep very briefly while waiting for a thread to get somewhere.
-    def wait
-      sleep 0.0001
-    end
-
-    def _sleeping?
-      if @state == :sleeping
-        # There's a very small period of time between when the Worker marks
-        # itself as sleeping and when it actually goes to sleep. Only report
-        # true when we're certain the thread is sleeping.
-        wait until @thread.status == 'sleep'
-        true
-      end
-    end
-
     def work_loop
-      loop do
-        cycle = nil
-
-        if Que.mode == :async
-          time   = Time.now
-          result = Job.work(queue)
-
-          case result[:event]
-          when :job_unavailable
-            cycle = false
-            result[:level] = :debug
-          when :job_race_condition
-            cycle = true
-            result[:level] = :debug
-          when :job_worked
-            cycle = true
-            result[:elapsed] = (Time.now - time).round(5)
-          when :job_errored
-            # For PG::Errors, assume we had a problem reaching the database, and
-            # don't hit it again right away.
-            cycle = !result[:error].is_a?(PG::Error)
-            result[:error] = {:class => result[:error].class.to_s, :message => result[:error].message}
-          else
-            raise "Unknown Event: #{result[:event].inspect}"
-          end
-
-          Que.log(result)
+      # Blocks until a job of the appropriate priority is available.
+      # `fetch_next_metajob` normally returns a job to be processed.
+      # If the queue is shutting down it will return false, which breaks the loop and
+      # lets the thread finish.
+      while (metajob = fetch_next_metajob) != false
+        # If metajob is nil instead of false, we've hit a rare race condition where
+        # there was a job in the buffer when the worker code checked, but the job was
+        # picked up by the time we got around to shifting it off the buffer.
+        # Letting this case go unhandled leads to worker threads exiting pre-maturely, so
+        # we check explicitly and continue the loop.
+        next if metajob.nil?
+        id = metajob.id
+
+        Que.internal_log(:worker_received_job, self) { {id: id} }
+
+        if Que.execute(:check_job, [id]).first
+          Que.recursively_freeze(metajob.job)
+          Que.internal_log(:worker_fetched_job, self) { {id: id} }
+
+          work_job(metajob)
+        else
+          # The job was locked but doesn't exist anymore, due to a race
+          # condition that exists because advisory locks don't obey MVCC. Not
+          # necessarily a problem, but if it happens a lot it may be meaningful.
+          Que.internal_log(:worker_job_lock_race_condition, self) { {id: id} }
         end
 
-        synchronize { @state = :sleeping unless cycle || @stop }
-        sleep if @state == :sleeping
-        break if @stop
+        Que.internal_log(:worker_pushing_finished_job, self) { {id: id} }
+
+        @result_queue.push(
+          metajob: metajob,
+          message_type: :job_finished,
+        )
       end
-    ensure
-      @state = :stopped
     end
 
-    # Setting Que.wake_interval = nil should ensure that the wrangler thread
-    # doesn't wake up a worker again, even if it's currently sleeping for a
-    # set period. So, we double-check that @wake_interval is set before waking
-    # a worker, and make sure to wake up the wrangler when @wake_interval is
-    # changed in Que.wake_interval= below.
-    @wake_interval = 5
-
-    # Four workers is a sensible default for most use cases.
-    @worker_count = 4
+    def fetch_next_metajob
+      @job_buffer.shift(*priority)
+    end
 
-    class << self
-      attr_reader :mode, :wake_interval, :worker_count
-      attr_accessor :queue_name
+    def work_job(metajob)
+      job      = metajob.job
+      start    = Time.now
+      klass    = Que.constantize(job.fetch(:job_class))
+      instance = klass.new(job)
 
-      # In order to work in a forking webserver, we need to be able to accept
-      # worker_count and wake_interval settings without actually instantiating
-      # the relevant threads until the mode is actually set to :async in a
-      # post-fork hook (since forking will kill any running background threads).
+      Que.run_job_middleware(instance) { instance.tap(&:_run) }
 
-      def mode=(mode)
-        Que.log :event => 'mode_change', :value => mode.to_s
-        @mode = mode
+      elapsed = Time.now - start
 
-        if mode == :async
-          set_up_workers
-          wrangler
+      log_level =
+        if instance.que_error
+          :error
+        else
+          instance.log_level(elapsed)
         end
-      end
-
-      def worker_count=(count)
-        Que.log :event => 'worker_count_change', :value => count.to_s
-        @worker_count = count
-        set_up_workers if mode == :async
-      end
-
-      def workers
-        @workers ||= []
-      end
-
-      def wake_interval=(interval)
-        @wake_interval = interval
-        wrangler.wakeup if mode == :async
-      end
 
-      def wake!
-        workers.find(&:wake!)
-      end
+      if VALID_LOG_LEVELS.include?(log_level)
+        log_message = {
+          level: log_level,
+          job_id: metajob.id,
+          elapsed: elapsed,
+        }
+
+        if error = instance.que_error
+          log_message[:event] = :job_errored
+          log_message[:error] = "#{error.class}: #{error.message}".slice(0, 500)
+        else
+          log_message[:event] = :job_worked
+        end
 
-      def wake_all!
-        workers.each(&:wake!)
+        Que.log(**log_message)
       end
 
-      private
-
-      def set_up_workers
-        if worker_count > workers.count
-          workers.push(*(worker_count - workers.count).times.map{new(queue_name || '')})
-        elsif worker_count < workers.count
-          workers.pop(workers.count - worker_count).each(&:stop).each(&:wait_until_stopped)
+      instance
+    rescue => error
+      Que.log(
+        level: :debug,
+        event: :job_errored,
+        job_id: metajob.id,
+        error: {
+          class:   error.class.to_s,
+          message: error.message,
+          backtrace: (error.backtrace || []).join("\n").slice(0, 10000),
+        },
+      )
+
+      Que.notify_error(error)
+
+      begin
+        # If the Job class couldn't be resolved, use the default retry
+        # backoff logic in Que::Job.
+        job_class = (klass && klass <= Job) ? klass : Job
+
+        error_count = job.fetch(:error_count) + 1
+
+        max_retry_count = job_class.resolve_que_setting(:maximum_retry_count)
+
+        if max_retry_count && error_count > max_retry_count
+          Que.execute :expire_job, [job.fetch(:id)]
+        else
+          delay =
+            job_class.
+            resolve_que_setting(
+              :retry_interval,
+              error_count,
+            )
+
+          Que.execute :set_error, [
+            delay,
+            "#{error.class}: #{error.message}".slice(0, 500),
+            (error.backtrace || []).join("\n").slice(0, 10000),
+            job.fetch(:id),
+          ]
         end
+      rescue
+        # If we can't reach the database for some reason, too bad, but
+        # don't let it crash the work loop.
       end
 
-      def wrangler
-        @wrangler ||= Thread.new do
-          loop do
-            sleep(*@wake_interval)
-            wake! if @wake_interval && mode == :async
-          end
-        end
-      end
+      error
     end
   end
 end