que 1.0.0.beta → 1.0.0.beta5

data/lib/que/active_record/connection.rb

@@ -17,17 +17,17 @@ module Que
 
         # Use Rails' executor (if present) to make sure that the connection
         # we're using isn't taken from us while the block runs. See
-        # https://github.com/chanks/que/issues/166#issuecomment-274218910
-        def wrap_in_rails_executor
+        # https://github.com/que-rb/que/issues/166#issuecomment-274218910
+        def wrap_in_rails_executor(&block)
           if defined?(::Rails.application.executor)
-            ::Rails.application.executor.wrap { yield }
+            ::Rails.application.executor.wrap(&block)
           else
             yield
           end
         end
       end
 
-      module Middleware
+      module JobMiddleware
         class << self
           def call(job)
             yield

data/lib/que/active_record/model.rb

@@ -3,7 +3,7 @@
 module Que
   module ActiveRecord
     class Model < ::ActiveRecord::Base
-      self.table_name = :que_jobs
+      self.table_name = 'public.que_jobs'
 
       t = arel_table
 
@@ -16,11 +16,11 @@ module Que
       scope :finished,     -> { where(t[:finished_at].not_eq(nil)) }
       scope :not_finished, -> { where(t[:finished_at].eq(nil)) }
 
-      scope :scheduled,     -> { where(t[:run_at].gt("now()")) }
-      scope :not_scheduled, -> { where(t[:run_at].lteq("now()")) }
+      scope :scheduled,     -> { where(t[:run_at].gt  (Arel.sql("now()"))) }
+      scope :not_scheduled, -> { where(t[:run_at].lteq(Arel.sql("now()"))) }
 
       scope :ready,     -> { not_errored.not_expired.not_finished.not_scheduled }
-      scope :not_ready, -> { where(t[:error_count].gt(0).or(t[:expired_at].not_eq(nil)).or(t[:finished_at].not_eq(nil)).or(t[:run_at].gt("now()"))) }
+      scope :not_ready, -> { where(t[:error_count].gt(0).or(t[:expired_at].not_eq(nil)).or(t[:finished_at].not_eq(nil)).or(t[:run_at].gt(Arel.sql("now()")))) }
 
       class << self
         def by_job_class(job_class)

data/lib/que/connection.rb

@@ -30,8 +30,11 @@ module Que
         when self
           conn
         when PG::Connection
-          conn.instance_variable_get(:@que_wrapper) ||
-          conn.instance_variable_set(:@que_wrapper, new(conn))
+          if conn.instance_variable_defined?(:@que_wrapper)
+            conn.instance_variable_get(:@que_wrapper)
+          else
+            conn.instance_variable_set(:@que_wrapper, new(conn))
+          end
         else
           raise Error, "Unsupported input for Connection.wrap: #{conn.class}"
         end
@@ -43,7 +46,7 @@ module Que
       @prepared_statements = Set.new
     end
 
-    def execute(command, params = nil)
+    def execute(command, params = [])
       sql =
         case command
         when Symbol then SQL[command]
@@ -51,8 +54,17 @@ module Que
         else raise Error, "Bad command! #{command.inspect}"
         end
 
-      params = convert_params(params) if params
-      result = execute_sql(sql, params)
+      params = convert_params(params)
+
+      result =
+        Que.run_sql_middleware(sql, params) do
+          # Some versions of the PG gem dislike an empty/nil params argument.
+          if params.empty?
+            wrapped_connection.async_exec(sql)
+          else
+            wrapped_connection.async_exec(sql, params)
+          end
+        end
 
       Que.internal_log :connection_execute, self do
         {
@@ -107,6 +119,10 @@ module Que
       loop { break if next_notification.nil? }
     end
 
+    def server_version
+      wrapped_connection.server_version
+    end
+
     def in_transaction?
       wrapped_connection.transaction_status != ::PG::PQTRANS_IDLE
     end
@@ -129,21 +145,24 @@ module Que
       end
     end
 
-    def execute_sql(sql, params)
-      # Some versions of the PG gem dislike an empty/nil params argument.
-      if params && !params.empty?
-        wrapped_connection.async_exec(sql, params)
-      else
-        wrapped_connection.async_exec(sql)
-      end
-    end
-
     # Procs used to convert strings from Postgres into Ruby types.
     CAST_PROCS = {
       # Boolean
-      16   => 't'.method(:==),
+      16 => -> (value) {
+        case value
+        when String then value == 't'.freeze
+        else !!value
+        end
+      },
+
       # Timestamp with time zone
-      1184 => Time.method(:parse),
+      1184 => -> (value) {
+        case value
+        when Time then value
+        when String then Time.parse(value)
+        else raise "Unexpected time class: #{value.class} (#{value.inspect})"
+        end
+      }
     }
 
     # JSON, JSONB

data/lib/que/connection_pool.rb

@@ -63,8 +63,8 @@ module Que
 
     private
 
-    def sync
-      @mutex.synchronize { yield }
+    def sync(&block)
+      @mutex.synchronize(&block)
     end
 
     def current_connection

data/lib/que/job.rb

@@ -134,7 +134,7 @@ module Que
         Que.recursively_freeze(attrs)
 
         new(attrs).tap do |job|
-          Que.run_middleware(job) do
+          Que.run_job_middleware(job) do
             job._run(reraise_errors: true)
           end
         end

data/lib/que/{job_cache.rb → job_buffer.rb}

@@ -5,89 +5,88 @@
 # minimum priority, and stopping gracefully.
 
 module Que
-  class JobCache
+  class JobBuffer
     attr_reader :maximum_size, :minimum_size, :priority_queues
 
+    # Since we use a mutex, which is not reentrant, we have to be a little
+    # careful to not call a method that locks the mutex when we've already
+    # locked it. So, as a general rule, public methods handle locking the mutex
+    # when necessary, while private methods handle the actual underlying data
+    # changes. This lets us reuse those private methods without running into
+    # locking issues.
+
     def initialize(
       maximum_size:,
       minimum_size:,
       priorities:
     )
       @maximum_size = Que.assert(Integer, maximum_size)
-      Que.assert(maximum_size >= 0) { "maximum_size for a JobCache must be at least zero!" }
+      Que.assert(maximum_size >= 0) { "maximum_size for a JobBuffer must be at least zero!" }
 
       @minimum_size = Que.assert(Integer, minimum_size)
-      Que.assert(minimum_size >= 0) { "minimum_size for a JobCache must be at least zero!" }
+      Que.assert(minimum_size >= 0) { "minimum_size for a JobBuffer must be at least zero!" }
 
       Que.assert(minimum_size <= maximum_size) do
-        "minimum queue size (#{minimum_size}) is " \
-          "greater than the maximum queue size (#{maximum_size})!"
+        "minimum buffer size (#{minimum_size}) is " \
+          "greater than the maximum buffer size (#{maximum_size})!"
       end
 
-      @stop    = false
-      @array   = []
-      @monitor = Monitor.new # TODO: Make this a mutex?
+      @stop  = false
+      @array = []
+      @mutex = Mutex.new
 
-      # Make sure that priority = nil sorts highest.
       @priority_queues = Hash[
-        priorities.sort_by{|p| p || Float::INFINITY}.map do |p|
-          [p, PriorityQueue.new(priority: p, job_cache: self)]
+        # Make sure that priority = nil sorts highest.
+        priorities.sort_by{|p| p || MAXIMUM_PRIORITY}.map do |p|
+          [p, PriorityQueue.new(priority: p, job_buffer: self)]
         end
       ].freeze
     end
 
     def push(*metajobs)
-      Que.internal_log(:job_cache_push, self) do
+      Que.internal_log(:job_buffer_push, self) do
         {
           maximum_size:  maximum_size,
           ids:           metajobs.map(&:id),
-          current_queue: @array,
+          current_queue: to_a,
         }
       end
 
       sync do
-        return metajobs if stopping?
+        return metajobs if _stopping?
 
-        @array.push(*metajobs).sort!
+        @array.concat(metajobs).sort!
 
         # Relying on the hash's contents being sorted, here.
         priority_queues.reverse_each do |_, pq|
-          pq.waiting_count.times do
-            job = shift_job(pq.priority)
-            break if job.nil?
-            pq.push(job)
+          pq.populate do
+            _shift_job(pq.priority)
           end
         end
 
-        # If we passed the maximum queue size, drop the lowest sort keys and
+        # If we passed the maximum buffer size, drop the lowest sort keys and
         # return their ids to be unlocked.
-        overage = -cache_space
+        overage = -_buffer_space
         pop(overage) if overage > 0
       end
     end
 
     def shift(priority = nil)
       queue = priority_queues.fetch(priority) { raise Error, "not a permitted priority! #{priority}" }
-      queue.pop
+      queue.pop || shift_job(priority)
     end
 
     def shift_job(priority = nil)
-      sync do
-        if stopping?
-          false
-        elsif (job = @array.first) && job.priority_sufficient?(priority)
-          @array.shift
-        end
-      end
+      sync { _shift_job(priority) }
     end
 
     def accept?(metajobs)
-      return [] if stopping?
-
       metajobs.sort!
 
       sync do
-        start_index = cache_space
+        return [] if _stopping?
+
+        start_index = _buffer_space
         final_index = metajobs.length - 1
 
         return metajobs if start_index > final_index
@@ -106,10 +105,6 @@ module Que
       end
     end
 
-    def jobs_needed?
-      minimum_size > size
-    end
-
     def waiting_count
       count = 0
       priority_queues.each_value do |pq|
@@ -126,7 +121,7 @@ module Que
         count = pq.waiting_count
 
         if lowest_priority
-          count += cache_space
+          count += buffer_space
           lowest_priority = false
         end
 
@@ -136,14 +131,12 @@ module Que
       hash
     end
 
-    def cache_space
-      sync do
-        maximum_size - size
-      end
+    def buffer_space
+      sync { _buffer_space }
     end
 
     def size
-      sync { @array.size }
+      sync { _size }
     end
 
     def to_a
@@ -156,73 +149,98 @@ module Que
     end
 
     def clear
-      sync { pop(size) }
+      sync { pop(_size) }
     end
 
     def stopping?
-      sync { !!@stop }
+      sync { _stopping? }
+    end
+
+    def job_available?(priority)
+      (job = @array.first) && job.priority_sufficient?(priority)
     end
 
     private
 
+    def _buffer_space
+      maximum_size - _size
+    end
+
     def pop(count)
       @array.pop(count)
     end
 
-    def sync
-      @monitor.synchronize { yield }
+    def _shift_job(priority)
+      if _stopping?
+        false
+      elsif (job = @array.first) && job.priority_sufficient?(priority)
+        @array.shift
+      end
+    end
+
+    def _size
+      @array.size
+    end
+
+    def _stopping?
+      !!@stop
+    end
+
+    def sync(&block)
+      @mutex.synchronize(&block)
     end
 
     # A queue object dedicated to a specific worker priority. It's basically a
     # Queue object from the standard library, but it's able to reach into the
-    # JobCache's cache in order to satisfy a pop.
+    # JobBuffer's buffer in order to satisfy a pop.
     class PriorityQueue
-      attr_reader :job_cache, :priority
+      attr_reader :job_buffer, :priority, :mutex
 
       def initialize(
-        job_cache:,
+        job_buffer:,
         priority:
       )
-        @job_cache = job_cache
-        @priority  = priority
-        @waiting   = 0
-        @stopping  = false
-        @items     = [] # Items pending distribution to waiting threads.
-        @monitor   = Monitor.new
-        @cv        = Monitor::ConditionVariable.new(@monitor)
+        @job_buffer = job_buffer
+        @priority   = priority
+        @waiting    = 0
+        @stopping   = false
+        @items      = [] # Items pending distribution to waiting threads.
+        @mutex      = Mutex.new
+        @cv         = ConditionVariable.new
       end
 
       def pop
         sync do
           loop do
-            return false if @stopping
-
-            if item = @items.pop
+            if @stopping
+              return false
+            elsif item = @items.pop
               return item
+            elsif job_buffer.job_available?(priority)
+              return false
             end
 
-            job = job_cache.shift_job(priority)
-            return job unless job.nil? # False means we're stopping.
-
             @waiting += 1
-            @cv.wait
+            @cv.wait(mutex)
             @waiting -= 1
           end
         end
       end
 
-      def push(item)
+      def stop
         sync do
-          Que.assert(waiting_count > 0)
-          @items << item
-          @cv.signal
+          @stopping = true
+          @cv.broadcast
         end
       end
 
-      def stop
+      def populate
         sync do
-          @stopping = true
-          @cv.broadcast
+          waiting_count.times do
+            job = yield
+            break if job.nil? # False would mean we're stopping.
+            _push(job)
+          end
         end
       end
 
@@ -232,8 +250,14 @@ module Que
 
       private
 
-      def sync
-        @monitor.synchronize { yield }
+      def sync(&block)
+        mutex.synchronize(&block)
+      end
+
+      def _push(item)
+        Que.assert(waiting_count > 0)
+        @items << item
+        @cv.signal
       end
     end
   end

data/lib/que/job_methods.rb

@@ -65,6 +65,10 @@ module Que
       raise error if reraise_errors
     end
 
+    def log_level(elapsed)
+      :debug
+    end
+
     private
 
     # This method defines the object on which the various job helper methods are