que 1.0.0.beta → 1.0.0.beta5

data/lib/que/locker.rb

@@ -19,35 +19,17 @@ module Que
     %{
       DELETE FROM public.que_lockers
       WHERE pid = pg_backend_pid()
-      OR pid NOT IN (SELECT pid FROM pg_stat_activity)
+      OR NOT EXISTS (SELECT 1 FROM pg_stat_activity WHERE pid = public.que_lockers.pid)
     }
 
   SQL[:register_locker] =
     %{
-      INSERT INTO public.que_lockers
-      (
-        pid,
-        worker_count,
-        worker_priorities,
-        ruby_pid,
-        ruby_hostname,
-        listening,
-        queues
-      )
-      VALUES
-      (
-        pg_backend_pid(),
-        $1::integer,
-        $2::integer[],
-        $3::integer,
-        $4::text,
-        $5::boolean,
-        $6::text[]
-      )
+      INSERT INTO public.que_lockers (pid, worker_count, worker_priorities, ruby_pid, ruby_hostname, listening, queues)
+      VALUES (pg_backend_pid(), $1::integer, $2::integer[], $3::integer, $4::text, $5::boolean, $6::text[])
     }
 
   class Locker
-    attr_reader :thread, :workers, :job_cache, :locks
+    attr_reader :thread, :workers, :job_buffer, :locks
 
     MESSAGE_RESOLVERS = {}
     RESULT_RESOLVERS  = {}
@@ -55,31 +37,29 @@ module Que
     MESSAGE_RESOLVERS[:job_available] =
       -> (messages) {
         metajobs = messages.map { |key| Metajob.new(key) }
-        push_jobs(lock_jobs(job_cache.accept?(metajobs)))
+        push_jobs(lock_jobs(job_buffer.accept?(metajobs)))
       }
 
     RESULT_RESOLVERS[:job_finished] =
       -> (messages) { finish_jobs(messages.map{|m| m.fetch(:metajob)}) }
 
-    DEFAULT_POLL_INTERVAL      = 5.0
-    DEFAULT_WAIT_PERIOD        = 50
-    DEFAULT_MINIMUM_QUEUE_SIZE = 2
-    DEFAULT_MAXIMUM_QUEUE_SIZE = 8
-    DEFAULT_WORKER_COUNT       = 6
-    DEFAULT_WORKER_PRIORITIES  = [10, 30, 50].freeze
+    DEFAULT_POLL_INTERVAL       = 5.0
+    DEFAULT_WAIT_PERIOD         = 50
+    DEFAULT_MINIMUM_BUFFER_SIZE = 2
+    DEFAULT_MAXIMUM_BUFFER_SIZE = 8
+    DEFAULT_WORKER_PRIORITIES   = [10, 30, 50, nil, nil, nil].freeze
 
     def initialize(
-      queues:             [Que.default_queue],
-      connection:         nil,
-      listen:             true,
-      poll:               true,
-      poll_interval:      DEFAULT_POLL_INTERVAL,
-      wait_period:        DEFAULT_WAIT_PERIOD,
-      maximum_queue_size: DEFAULT_MAXIMUM_QUEUE_SIZE,
-      minimum_queue_size: DEFAULT_MINIMUM_QUEUE_SIZE,
-      worker_count:       DEFAULT_WORKER_COUNT,
-      worker_priorities:  DEFAULT_WORKER_PRIORITIES,
-      on_worker_start:    nil
+      queues:              [Que.default_queue],
+      connection_url:      nil,
+      listen:              true,
+      poll:                true,
+      poll_interval:       DEFAULT_POLL_INTERVAL,
+      wait_period:         DEFAULT_WAIT_PERIOD,
+      maximum_buffer_size: DEFAULT_MAXIMUM_BUFFER_SIZE,
+      minimum_buffer_size: DEFAULT_MINIMUM_BUFFER_SIZE,
+      worker_priorities:   DEFAULT_WORKER_PRIORITIES,
+      on_worker_start:     nil
     )
 
       # Sanity-check all our arguments, since some users may instantiate Locker
@@ -89,34 +69,32 @@ module Que
 
       Que.assert Numeric, poll_interval
       Que.assert Numeric, wait_period
-      Que.assert Integer, worker_count
 
       Que.assert Array, worker_priorities
-      worker_priorities.each { |p| Que.assert(Integer, p) }
-
-      all_worker_priorities = worker_priorities.values_at(0...worker_count)
+      worker_priorities.each { |p| Que.assert([Integer, NilClass], p) }
 
-      # We use a JobCache to track jobs and pass them to workers, and a
+      # We use a JobBuffer to track jobs and pass them to workers, and a
       # ResultQueue to receive messages from workers.
-      @job_cache = JobCache.new(
-        maximum_size: maximum_queue_size,
-        minimum_size: minimum_queue_size,
-        priorities:   all_worker_priorities.uniq,
+      @job_buffer = JobBuffer.new(
+        maximum_size: maximum_buffer_size,
+        minimum_size: minimum_buffer_size,
+        priorities:   worker_priorities.uniq,
       )
 
       @result_queue = ResultQueue.new
 
+      @stop = false
+
       Que.internal_log :locker_instantiate, self do
         {
-          queues:             queues,
-          listen:             listen,
-          poll:               poll,
-          poll_interval:      poll_interval,
-          wait_period:        wait_period,
-          maximum_queue_size: maximum_queue_size,
-          minimum_queue_size: minimum_queue_size,
-          worker_count:       worker_count,
-          worker_priorities:  worker_priorities,
+          queues:              queues,
+          listen:              listen,
+          poll:                poll,
+          poll_interval:       poll_interval,
+          wait_period:         wait_period,
+          maximum_buffer_size: maximum_buffer_size,
+          minimum_buffer_size: minimum_buffer_size,
+          worker_priorities:   worker_priorities,
         }
       end
 
@@ -126,16 +104,11 @@ module Que
       @queue_names = queues.is_a?(Hash) ? queues.keys : queues
       @wait_period = wait_period.to_f / 1000 # Milliseconds to seconds.
 
-      # If the worker_count exceeds the array of priorities it'll result in
-      # extra workers that will work jobs of any priority. For example, the
-      # default worker_count of 6 and the default worker priorities of [10, 30,
-      # 50] will result in three workers that only work jobs that meet those
-      # priorities, and three workers that will work any job.
       @workers =
-        all_worker_priorities.map do |priority|
+        worker_priorities.map do |priority|
           Worker.new(
             priority:       priority,
-            job_cache:      @job_cache,
+            job_buffer:     @job_buffer,
             result_queue:   @result_queue,
             start_callback: on_worker_start,
           )
@@ -144,18 +117,46 @@ module Que
       # To prevent race conditions, let every worker get into a ready state
       # before starting up the locker thread.
       loop do
-        break if job_cache.waiting_count == workers.count
+        break if job_buffer.waiting_count == workers.count
         sleep 0.001
       end
 
-      pool =
-        if connection
-          # Wrap the given connection in a dummy connection pool.
-          ConnectionPool.new { |&block| block.call(connection) }
+      # If we weren't passed a specific connection_url, borrow a connection from
+      # the pool and derive the connection string from it.
+      connection_args =
+        if connection_url
+          uri = URI.parse(connection_url)
+
+          opts =
+            {
+              host:     uri.host,
+              user:     uri.user,
+              password: uri.password,
+              port:     uri.port || 5432,
+              dbname:   uri.path[1..-1],
+            }
+
+          if uri.query
+            opts.merge!(Hash[uri.query.split("&").map{|s| s.split('=')}.map{|a,b| [a.to_sym, b]}])
+          end
+
+          opts
         else
-          Que.pool
+          Que.pool.checkout do |conn|
+            c = conn.wrapped_connection
+
+            {
+              host:     c.host,
+              user:     c.user,
+              password: c.pass,
+              port:     c.port,
+              dbname:   c.db,
+            }
+          end
         end
 
+      @connection = Que::Connection.wrap(PG::Connection.open(connection_args))
+
       @thread =
         Thread.new do
           # An error causing this thread to exit is a bug in Que, which we want
@@ -165,47 +166,35 @@ module Que
           # Give this thread priority, so it can promptly respond to NOTIFYs.
           Thread.current.priority = 1
 
-          pool.checkout do |connection|
-            original_application_name =
-              connection.
-              execute("SHOW application_name").
-              first.
-              fetch(:application_name)
-
-            begin
-              @connection = connection
-
-              connection.execute(
+          begin
+            unless connection_args.has_key?(:application_name)
+              @connection.execute(
                 "SELECT set_config('application_name', $1, false)",
-                ["Que Locker: #{connection.backend_pid}"]
+                ["Que Locker: #{@connection.backend_pid}"]
               )
+            end
 
-              Poller.setup(connection)
+            Poller.setup(@connection)
 
+            @listener =
               if listen
-                @listener = Listener.new(connection: connection)
+                Listener.new(connection: @connection)
               end
 
+            @pollers =
               if poll
-                @pollers =
-                  queues.map do |queue, interval|
-                    Poller.new(
-                      connection:    connection,
-                      queue:         queue,
-                      poll_interval: interval || poll_interval,
-                    )
-                  end
+                queues.map do |queue, interval|
+                  Poller.new(
+                    connection:    @connection,
+                    queue:         queue,
+                    poll_interval: interval || poll_interval,
+                  )
+                end
               end
 
-              work_loop
-            ensure
-              connection.execute(
-                "SELECT set_config('application_name', $1, false)",
-                [original_application_name]
-              )
-
-              Poller.cleanup(connection)
-            end
+            work_loop
+          ensure
+            @connection.wrapped_connection.close
           end
         end
     end
@@ -215,7 +204,7 @@ module Que
     end
 
     def stop
-      @job_cache.stop
+      @job_buffer.stop
       @stop = true
     end
 
@@ -249,17 +238,7 @@ module Que
       begin
         @listener.listen if @listener
 
-        # A previous locker that didn't exit cleanly may have left behind
-        # a bad locker record, so clean up before registering.
-        connection.execute :clean_lockers
-        connection.execute :register_locker, [
-          @workers.count,
-          "{#{@workers.map(&:priority).map{|p| p || 'NULL'}.join(',')}}",
-          Process.pid,
-          CURRENT_HOSTNAME,
-          !!@listener,
-          "{\"#{@queue_names.join('","')}\"}",
-        ]
+        startup
 
         {} while cycle
 
@@ -268,11 +247,7 @@ module Que
           event: :locker_stop,
         )
 
-        unlock_jobs(@job_cache.clear)
-
-        @workers.each(&:wait_until_stopped)
-
-        handle_results
+        shutdown
       ensure
         connection.execute :clean_lockers
 
@@ -280,6 +255,20 @@ module Que
       end
     end
 
+    def startup
+      # A previous locker that didn't exit cleanly may have left behind
+      # a bad locker record, so clean up before registering.
+      connection.execute :clean_lockers
+      connection.execute :register_locker, [
+        @workers.count,
+        "{#{@workers.map(&:priority).map{|p| p || 'NULL'}.join(',')}}",
+        Process.pid,
+        CURRENT_HOSTNAME,
+        !!@listener,
+        "{\"#{@queue_names.join('","')}\"}",
+      ]
+    end
+
     def cycle
       # Poll at the start of a cycle, so that when the worker starts up we can
       # load up the queue with jobs immediately.
@@ -300,31 +289,73 @@ module Que
       !@stop
     end
 
+    def shutdown
+      unlock_jobs(@job_buffer.clear)
+      wait_for_shutdown
+      handle_results
+    end
+
+    def wait_for_shutdown
+      @workers.each(&:wait_until_stopped)
+    end
+
     def poll
       # Only poll when there are pollers to use (that is, when polling is
-      # enabled) and when the local queue has dropped below the configured
-      # minimum size.
-      return unless pollers && job_cache.jobs_needed?
+      # enabled).
+      return unless pollers
 
-      pollers.each do |poller|
-        priorities = job_cache.available_priorities
-        break if priorities.empty?
+      # Figure out what job priorities we have to fill.
+      priorities = job_buffer.available_priorities
+
+      # Only poll when there are workers ready for jobs.
+      return if priorities.empty?
+
+      all_metajobs = []
 
-        Que.internal_log(:locker_polling, self) { {priorities: priorities, held_locks: @locks.to_a, queue: poller.queue} }
+      pollers.each do |poller|
+        Que.internal_log(:locker_polling, self) {
+          {
+            priorities: priorities,
+            held_locks: @locks.to_a,
+            queue: poller.queue,
+          }
+        }
 
         if metajobs = poller.poll(priorities: priorities, held_locks: @locks)
+          metajobs.sort!
+          all_metajobs.concat(metajobs)
+
+          # Update the desired priorities list to take the priorities that we
+          # just retrieved into account.
           metajobs.each do |metajob|
-            mark_id_as_locked(metajob.id)
+            job_priority = metajob.job.fetch(:priority)
+
+            priorities.each do |priority, count|
+              if job_priority <= priority
+                new_priority = count - 1
+
+                if new_priority <= 0
+                  priorities.delete(priority)
+                else
+                  priorities[priority] = new_priority
+                end
+
+                break
+              end
+            end
           end
 
-          push_jobs(metajobs)
+          break if priorities.empty?
         end
       end
+
+      all_metajobs.each { |metajob| mark_id_as_locked(metajob.id) }
+      push_jobs(all_metajobs)
     end
 
     def wait
-      if @listener
-        @listener.wait_for_grouped_messages(@wait_period).each do |type, messages|
+      if l = @listener
+        l.wait_for_grouped_messages(@wait_period).each do |type, messages|
           if resolver = MESSAGE_RESOLVERS[type]
             instance_exec messages, &resolver
           else
@@ -353,7 +384,7 @@ module Que
       metajobs.reject! { |m| @locks.include?(m.id) }
       return metajobs if metajobs.empty?
 
-      ids = metajobs.map{|m| m.id.to_i}
+      ids = metajobs.map { |m| m.id.to_i }
 
       Que.internal_log :locker_locking, self do
         {
@@ -362,12 +393,12 @@ module Que
         }
       end
 
+      materalize_cte = connection.server_version >= 12_00_00
+
       jobs =
         connection.execute \
           <<-SQL
-            WITH jobs AS (
-              SELECT * FROM que_jobs WHERE id IN (#{ids.join(', ')})
-            )
+            WITH jobs AS #{materalize_cte ? 'MATERIALIZED' : ''} (SELECT * FROM que_jobs WHERE id IN (#{ids.join(', ')}))
             SELECT * FROM jobs WHERE pg_try_advisory_lock(id)
           SQL
 
@@ -408,12 +439,12 @@ module Que
 
       good, bad = metajobs.partition{|mj| verified_ids.include?(mj.id)}
 
-      displaced = @job_cache.push(*good) || []
-
-      # Unlock any low-importance jobs the new ones may displace.
-      if bad.any? || displaced.any?
-        unlock_jobs(bad + displaced)
+      # Need to unlock any low-importance jobs the new ones may displace.
+      if displaced = @job_buffer.push(*good)
+        bad.concat(displaced)
       end
+
+      unlock_jobs(bad)
     end
 
     def finish_jobs(metajobs)

data/lib/que/poller.rb

@@ -144,7 +144,7 @@ module Que
 
       return unless should_poll?
 
-      expected_count = priorities.inject(0){|s,(p,c)| s + c}
+      expected_count = priorities.inject(0){|s,(_,c)| s + c}
 
       jobs =
         connection.execute_prepared(

data/lib/que/rails/railtie.rb

@@ -2,12 +2,10 @@
 
 module Que
   module Rails
-    class Railtie < Rails::Railtie
+    class Railtie < ::Rails::Railtie
       config.que = Que
 
-      Que.run_asynchronously = true if Rails.env.test?
-
-      Que.logger     = proc { Rails.logger }
+      Que.logger     = proc { ::Rails.logger }
       Que.connection = ::ActiveRecord if defined? ::ActiveRecord
     end
   end

data/lib/que/result_queue.rb

@@ -28,8 +28,8 @@ module Que
 
     private
 
-    def sync
-      @mutex.synchronize { yield }
+    def sync(&block)
+      @mutex.synchronize(&block)
     end
   end
 end

data/lib/que/sequel/model.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+::Sequel.extension :pg_json_ops
+
 module Que
   module Sequel
     QUALIFIED_TABLE = ::Sequel.qualify(:public, :que_jobs)
@@ -7,10 +9,10 @@ module Que
     class Model < ::Sequel::Model(QUALIFIED_TABLE)
       dataset_module do
         conditions = {
-          errored:   ::Sequel.qualify(QUALIFIED_TABLE, :error_count) > 0,
-          expired:   ::Sequel.~(::Sequel.qualify(QUALIFIED_TABLE, :expired_at)  => nil),
-          finished:  ::Sequel.~(::Sequel.qualify(QUALIFIED_TABLE, :finished_at) => nil),
-          scheduled: ::Sequel.qualify(QUALIFIED_TABLE, :run_at) > ::Sequel::CURRENT_TIMESTAMP,
+          errored:   QUALIFIED_TABLE[:error_count] > 0,
+          expired:   QUALIFIED_TABLE[:expired_at] !~ nil,
+          finished:  QUALIFIED_TABLE[:finished_at] !~ nil,
+          scheduled: QUALIFIED_TABLE[:run_at] > ::Sequel::CURRENT_TIMESTAMP,
         }
 
         conditions.each do |name, condition|
@@ -18,32 +20,28 @@ module Que
           subset :"not_#{name}", ~condition
         end
 
-        subset :ready,     conditions.values.map(&:~).inject{|a, b| a & b}
-        subset :not_ready, conditions.values.         inject{|a, b| a | b}
+        subset :ready,     conditions.values.map(&:~).inject(:&)
+        subset :not_ready, conditions.values.         inject(:|)
 
         def by_job_class(job_class)
           job_class = job_class.name if job_class.is_a?(Class)
           where(
-            ::Sequel.|(
-              {::Sequel.qualify(QUALIFIED_TABLE, :job_class) => job_class},
-              {
-                ::Sequel.qualify(QUALIFIED_TABLE, :job_class) => "ActiveJob::QueueAdapters::QueAdapter::JobWrapper",
-                ::Sequel.lit("public.que_jobs.args->0->>'job_class'") => job_class,
-              }
-            )
+            (QUALIFIED_TABLE[:job_class] =~ job_class) |
+              (QUALIFIED_TABLE[:job_class] =~ "ActiveJob::QueueAdapters::QueAdapter::JobWrapper") &
+              (QUALIFIED_TABLE[:args].pg_jsonb[0].get_text("job_class") =~ job_class)
           )
         end
 
         def by_queue(queue)
-          where(::Sequel.qualify(QUALIFIED_TABLE, :queue) => queue)
+          where(QUALIFIED_TABLE[:queue] => queue)
         end
 
         def by_tag(tag)
-          where(::Sequel.lit("public.que_jobs.data @> ?", JSON.dump(tags: [tag])))
+          where(QUALIFIED_TABLE[:data].pg_jsonb.contains(JSON.dump(tags: [tag])))
         end
 
         def by_args(*args)
-          where(::Sequel.lit("public.que_jobs.args @> ?", JSON.dump(args)))
+          where(QUALIFIED_TABLE[:args].pg_jsonb.contains(JSON.dump(args)))
         end
       end
     end

data/lib/que/utils/constantization.rb

@@ -4,7 +4,7 @@ module Que
   module Utils
     module Constantization
       def constantize(string)
-        assert String, string
+        Que.assert String, string
 
         if string.respond_to?(:constantize)
           string.constantize

data/lib/que/utils/logging.rb

@@ -5,7 +5,8 @@
 module Que
   module Utils
     module Logging
-      attr_accessor :logger, :log_formatter, :internal_logger
+      attr_accessor :logger, :internal_logger
+      attr_writer :log_formatter
 
       def log(event:, level: :info, **extra)
         data = _default_log_data

data/lib/que/utils/middleware.rb

@@ -5,27 +5,40 @@
 module Que
   module Utils
     module Middleware
-      def run_middleware(job, &block)
-        invoke_middleware(
-          middleware: middleware.dup,
-          job:        job,
-          block:      block,
-        )
-      end
+      TYPES = [
+        :job,
+        :sql,
+      ].freeze
+
+      TYPES.each do |type|
+        module_eval <<-CODE
+          def #{type}_middleware
+            @#{type}_middleware ||= []
+          end
 
-      def middleware
-        @middleware ||= []
+          def run_#{type}_middleware(*args)
+            m = #{type}_middleware
+
+            if m.empty?
+              yield
+            else
+              invoke_middleware(middleware: m.dup, args: args) { yield }
+            end
+          end
+        CODE
       end
 
       private
 
-      def invoke_middleware(middleware:, job:, block:)
+      def invoke_middleware(middleware:, args:, &block)
         if m = middleware.shift
-          m.call(job) do
-            invoke_middleware(middleware: middleware, job: job, block: block)
+          r = nil
+          m.call(*args) do
+            r = invoke_middleware(middleware: middleware, args: args, &block)
           end
+          r
         else
-          block.call
+          yield
         end
       end
     end

data/lib/que/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Que
-  VERSION = '1.0.0.beta'
+  VERSION = '1.0.0.beta5'
 end