que 1.0.0.beta2 → 1.0.0

data/lib/que/active_record/connection.rb

@@ -17,7 +17,7 @@ module Que
 
         # Use Rails' executor (if present) to make sure that the connection
         # we're using isn't taken from us while the block runs. See
-        # https://github.com/chanks/que/issues/166#issuecomment-274218910
+        # https://github.com/que-rb/que/issues/166#issuecomment-274218910
         def wrap_in_rails_executor(&block)
           if defined?(::Rails.application.executor)
             ::Rails.application.executor.wrap(&block)

data/lib/que/active_record/model.rb

@@ -3,7 +3,7 @@
 module Que
   module ActiveRecord
     class Model < ::ActiveRecord::Base
-      self.table_name = :que_jobs
+      self.table_name = 'public.que_jobs'
 
       t = arel_table
 

data/lib/que/connection.rb

@@ -119,6 +119,10 @@ module Que
       loop { break if next_notification.nil? }
     end
 
+    def server_version
+      wrapped_connection.server_version
+    end
+
     def in_transaction?
       wrapped_connection.transaction_status != ::PG::PQTRANS_IDLE
     end
@@ -152,7 +156,13 @@ module Que
       },
 
       # Timestamp with time zone
-      1184 => Time.method(:parse),
+      1184 => -> (value) {
+        case value
+        when Time then value
+        when String then Time.parse(value)
+        else raise "Unexpected time class: #{value.class} (#{value.inspect})"
+        end
+      }
     }
 
     # JSON, JSONB

data/lib/que/job_buffer.rb

@@ -59,10 +59,8 @@ module Que
 
         # Relying on the hash's contents being sorted, here.
         priority_queues.reverse_each do |_, pq|
-          pq.waiting_count.times do
-            job = _shift_job(pq.priority)
-            break if job.nil? # False would mean we're stopping.
-            pq.push(job)
+          pq.populate do
+            _shift_job(pq.priority)
           end
         end
 
@@ -75,7 +73,7 @@ module Que
 
     def shift(priority = nil)
       queue = priority_queues.fetch(priority) { raise Error, "not a permitted priority! #{priority}" }
-      queue.pop
+      queue.pop || shift_job(priority)
     end
 
     def shift_job(priority = nil)
@@ -107,10 +105,6 @@ module Que
       end
     end
 
-    def jobs_needed?
-      minimum_size > size
-    end
-
     def waiting_count
       count = 0
       priority_queues.each_value do |pq|
@@ -162,6 +156,10 @@ module Que
       sync { _stopping? }
     end
 
+    def job_available?(priority)
+      (job = @array.first) && job.priority_sufficient?(priority)
+    end
+
     private
 
     def _buffer_space
@@ -214,15 +212,14 @@ module Que
       def pop
         sync do
           loop do
-            return false if @stopping
-
-            if item = @items.pop
+            if @stopping
+              return false
+            elsif item = @items.pop
               return item
+            elsif job_buffer.job_available?(priority)
+              return false
             end
 
-            job = job_buffer.shift_job(priority)
-            return job unless job.nil? # False means we're stopping.
-
             @waiting += 1
             @cv.wait(mutex)
             @waiting -= 1
@@ -230,18 +227,20 @@ module Que
         end
       end
 
-      def push(item)
+      def stop
         sync do
-          Que.assert(waiting_count > 0)
-          @items << item
-          @cv.signal
+          @stopping = true
+          @cv.broadcast
         end
       end
 
-      def stop
+      def populate
         sync do
-          @stopping = true
-          @cv.broadcast
+          waiting_count.times do
+            job = yield
+            break if job.nil? # False would mean we're stopping.
+            _push(job)
+          end
         end
       end
 
@@ -254,6 +253,12 @@ module Que
       def sync(&block)
         mutex.synchronize(&block)
       end
+
+      def _push(item)
+        Que.assert(waiting_count > 0)
+        @items << item
+        @cv.signal
+      end
     end
   end
 end

data/lib/que/job_methods.rb

@@ -65,6 +65,10 @@ module Que
       raise error if reraise_errors
     end
 
+    def log_level(elapsed)
+      :debug
+    end
+
     private
 
     # This method defines the object on which the various job helper methods are

data/lib/que/locker.rb

@@ -47,8 +47,7 @@ module Que
     DEFAULT_WAIT_PERIOD         = 50
     DEFAULT_MINIMUM_BUFFER_SIZE = 2
     DEFAULT_MAXIMUM_BUFFER_SIZE = 8
-    DEFAULT_WORKER_COUNT        = 6
-    DEFAULT_WORKER_PRIORITIES   = [10, 30, 50].freeze
+    DEFAULT_WORKER_PRIORITIES   = [10, 30, 50, nil, nil, nil].freeze
 
     def initialize(
       queues:              [Que.default_queue],
@@ -59,7 +58,6 @@ module Que
       wait_period:         DEFAULT_WAIT_PERIOD,
       maximum_buffer_size: DEFAULT_MAXIMUM_BUFFER_SIZE,
       minimum_buffer_size: DEFAULT_MINIMUM_BUFFER_SIZE,
-      worker_count:        DEFAULT_WORKER_COUNT,
       worker_priorities:   DEFAULT_WORKER_PRIORITIES,
       on_worker_start:     nil
     )
@@ -71,19 +69,16 @@ module Que
 
       Que.assert Numeric, poll_interval
       Que.assert Numeric, wait_period
-      Que.assert Integer, worker_count
 
       Que.assert Array, worker_priorities
-      worker_priorities.each { |p| Que.assert(Integer, p) }
-
-      all_worker_priorities = worker_priorities.values_at(0...worker_count)
+      worker_priorities.each { |p| Que.assert([Integer, NilClass], p) }
 
       # We use a JobBuffer to track jobs and pass them to workers, and a
       # ResultQueue to receive messages from workers.
       @job_buffer = JobBuffer.new(
         maximum_size: maximum_buffer_size,
         minimum_size: minimum_buffer_size,
-        priorities:   all_worker_priorities.uniq,
+        priorities:   worker_priorities.uniq,
       )
 
       @result_queue = ResultQueue.new
@@ -99,7 +94,6 @@ module Que
           wait_period:         wait_period,
           maximum_buffer_size: maximum_buffer_size,
           minimum_buffer_size: minimum_buffer_size,
-          worker_count:        worker_count,
           worker_priorities:   worker_priorities,
         }
       end
@@ -110,13 +104,8 @@ module Que
       @queue_names = queues.is_a?(Hash) ? queues.keys : queues
       @wait_period = wait_period.to_f / 1000 # Milliseconds to seconds.
 
-      # If the worker_count exceeds the array of priorities it'll result in
-      # extra workers that will work jobs of any priority. For example, the
-      # default worker_count of 6 and the default worker priorities of [10, 30,
-      # 50] will result in three workers that only work jobs that meet those
-      # priorities, and three workers that will work any job.
       @workers =
-        all_worker_priorities.map do |priority|
+        worker_priorities.map do |priority|
           Worker.new(
             priority:       priority,
             job_buffer:     @job_buffer,
@@ -138,13 +127,20 @@ module Que
         if connection_url
           uri = URI.parse(connection_url)
 
-          {
-            host:     uri.host,
-            user:     uri.user,
-            password: uri.password,
-            port:     uri.port || 5432,
-            dbname:   uri.path[1..-1],
-          }.merge(Hash[uri.query.split("&").map{|s| s.split('=')}.map{|a,b| [a.to_sym, b]}])
+          opts =
+            {
+              host:     uri.host,
+              user:     uri.user,
+              password: uri.password,
+              port:     uri.port || 5432,
+              dbname:   uri.path[1..-1],
+            }
+
+          if uri.query
+            opts.merge!(Hash[uri.query.split("&").map{|s| s.split('=')}.map{|a,b| [a.to_sym, b]}])
+          end
+
+          opts
         else
           Que.pool.checkout do |conn|
             c = conn.wrapped_connection
@@ -305,12 +301,15 @@ module Que
 
     def poll
       # Only poll when there are pollers to use (that is, when polling is
-      # enabled) and when the local queue has dropped below the configured
-      # minimum size.
-      return unless pollers && job_buffer.jobs_needed?
+      # enabled).
+      return unless pollers
 
       # Figure out what job priorities we have to fill.
       priorities = job_buffer.available_priorities
+
+      # Only poll when there are workers ready for jobs.
+      return if priorities.empty?
+
       all_metajobs = []
 
       pollers.each do |poller|
@@ -394,10 +393,12 @@ module Que
         }
       end
 
+      materalize_cte = connection.server_version >= 12_00_00
+
       jobs =
         connection.execute \
           <<-SQL
-            WITH jobs AS (SELECT * FROM que_jobs WHERE id IN (#{ids.join(', ')}))
+            WITH jobs AS #{materalize_cte ? 'MATERIALIZED' : ''} (SELECT * FROM que_jobs WHERE id IN (#{ids.join(', ')}))
             SELECT * FROM jobs WHERE pg_try_advisory_lock(id)
           SQL
 

data/lib/que/rails/railtie.rb

@@ -5,8 +5,6 @@ module Que
     class Railtie < ::Rails::Railtie
       config.que = Que
 
-      Que.run_asynchronously = true if ::Rails.env.test?
-
       Que.logger     = proc { ::Rails.logger }
       Que.connection = ::ActiveRecord if defined? ::ActiveRecord
     end

data/lib/que/sequel/model.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+::Sequel.extension :pg_json_ops
+
 module Que
   module Sequel
     QUALIFIED_TABLE = ::Sequel.qualify(:public, :que_jobs)
@@ -7,10 +9,10 @@ module Que
     class Model < ::Sequel::Model(QUALIFIED_TABLE)
       dataset_module do
         conditions = {
-          errored:   ::Sequel.qualify(QUALIFIED_TABLE, :error_count) > 0,
-          expired:   ::Sequel.~(::Sequel.qualify(QUALIFIED_TABLE, :expired_at)  => nil),
-          finished:  ::Sequel.~(::Sequel.qualify(QUALIFIED_TABLE, :finished_at) => nil),
-          scheduled: ::Sequel.qualify(QUALIFIED_TABLE, :run_at) > ::Sequel::CURRENT_TIMESTAMP,
+          errored:   QUALIFIED_TABLE[:error_count] > 0,
+          expired:   QUALIFIED_TABLE[:expired_at] !~ nil,
+          finished:  QUALIFIED_TABLE[:finished_at] !~ nil,
+          scheduled: QUALIFIED_TABLE[:run_at] > ::Sequel::CURRENT_TIMESTAMP,
         }
 
         conditions.each do |name, condition|
@@ -18,32 +20,28 @@ module Que
           subset :"not_#{name}", ~condition
         end
 
-        subset :ready,     conditions.values.map(&:~).inject{|a, b| a & b}
-        subset :not_ready, conditions.values.         inject{|a, b| a | b}
+        subset :ready,     conditions.values.map(&:~).inject(:&)
+        subset :not_ready, conditions.values.         inject(:|)
 
         def by_job_class(job_class)
           job_class = job_class.name if job_class.is_a?(Class)
           where(
-            ::Sequel.|(
-              {::Sequel.qualify(QUALIFIED_TABLE, :job_class) => job_class},
-              {
-                ::Sequel.qualify(QUALIFIED_TABLE, :job_class) => "ActiveJob::QueueAdapters::QueAdapter::JobWrapper",
-                ::Sequel.lit("public.que_jobs.args->0->>'job_class'") => job_class,
-              }
-            )
+            (QUALIFIED_TABLE[:job_class] =~ job_class) |
+              (QUALIFIED_TABLE[:job_class] =~ "ActiveJob::QueueAdapters::QueAdapter::JobWrapper") &
+              (QUALIFIED_TABLE[:args].pg_jsonb[0].get_text("job_class") =~ job_class)
           )
         end
 
         def by_queue(queue)
-          where(::Sequel.qualify(QUALIFIED_TABLE, :queue) => queue)
+          where(QUALIFIED_TABLE[:queue] => queue)
         end
 
         def by_tag(tag)
-          where(::Sequel.lit("public.que_jobs.data @> ?", JSON.dump(tags: [tag])))
+          where(QUALIFIED_TABLE[:data].pg_jsonb.contains(JSON.dump(tags: [tag])))
         end
 
         def by_args(*args)
-          where(::Sequel.lit("public.que_jobs.args @> ?", JSON.dump(args)))
+          where(QUALIFIED_TABLE[:args].pg_jsonb.contains(JSON.dump(args)))
         end
       end
     end

data/lib/que/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Que
-  VERSION = '1.0.0.beta2'
+  VERSION = '1.0.0'
 end

data/lib/que/worker.rb

@@ -3,10 +3,14 @@
 # Workers wrap threads which continuously pull job pks from JobBuffer objects,
 # fetch and work those jobs, and export relevant data to ResultQueues.
 
+require 'set'
+
 module Que
   class Worker
     attr_reader :thread, :priority
 
+    VALID_LOG_LEVELS = [:debug, :info, :warn, :error, :fatal, :unknown].to_set.freeze
+
     SQL[:check_job] =
       %{
         SELECT 1 AS one
@@ -50,10 +54,17 @@ module Que
     private
 
     def work_loop
-      # Blocks until a job of the appropriate priority is available. If the
-      # queue is shutting down this will return nil, which breaks the loop and
+      # Blocks until a job of the appropriate priority is available.
+      # `fetch_next_metajob` normally returns a job to be processed.
+      # If the queue is shutting down it will return false, which breaks the loop and
       # lets the thread finish.
-      while metajob = fetch_next_metajob
+      while (metajob = fetch_next_metajob) != false
+        # If metajob is nil instead of false, we've hit a rare race condition where
+        # there was a job in the buffer when the worker code checked, but the job was
+        # picked up by the time we got around to shifting it off the buffer.
+        # Letting this case go unhandled leads to worker threads exiting pre-maturely, so
+        # we check explicitly and continue the loop.
+        next if metajob.nil?
         id = metajob.id
 
         Que.internal_log(:worker_received_job, self) { {id: id} }
@@ -91,20 +102,31 @@ module Que
 
       Que.run_job_middleware(instance) { instance.tap(&:_run) }
 
-      log_message = {
-        level: :debug,
-        job_id: metajob.id,
-        elapsed: (Time.now - start),
-      }
+      elapsed = Time.now - start
 
-      if error = instance.que_error
-        log_message[:event] = :job_errored
-        log_message[:error] = "#{error.class}: #{error.message}".slice(0, 500)
-      else
-        log_message[:event] = :job_worked
-      end
+      log_level =
+        if instance.que_error
+          :error
+        else
+          instance.log_level(elapsed)
+        end
+
+      if VALID_LOG_LEVELS.include?(log_level)
+        log_message = {
+          level: log_level,
+          job_id: metajob.id,
+          elapsed: elapsed,
+        }
 
-      Que.log(log_message)
+        if error = instance.que_error
+          log_message[:event] = :job_errored
+          log_message[:error] = "#{error.class}: #{error.message}".slice(0, 500)
+        else
+          log_message[:event] = :job_worked
+        end
+
+        Que.log(log_message)
+      end
 
       instance
     rescue => error

data/que.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.email         = ['christopher.m.hanks@gmail.com']
   spec.description   = %q{A job queue that uses PostgreSQL's advisory locks for speed and reliability.}
   spec.summary       = %q{A PostgreSQL-based Job Queue}
-  spec.homepage      = 'https://github.com/chanks/que'
+  spec.homepage      = 'https://github.com/que-rb/que'
   spec.license       = 'MIT'
 
   files_to_exclude = [
@@ -29,5 +29,5 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'bundler'
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: que
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta2
+  version: 1.0.0
 platform: ruby
 authors:
 - Chris Hanks
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-13 00:00:00.000000000 Z
+date: 2022-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
 description: A job queue that uses PostgreSQL's advisory locks for speed and reliability.
 email:
 - christopher.m.hanks@gmail.com
@@ -32,8 +32,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/tests.yml"
 - ".gitignore"
-- CHANGELOG.1.0.beta.md
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
@@ -41,21 +41,6 @@ files:
 - bin/command_line_interface.rb
 - bin/que
 - docs/README.md
-- docs/active_job.md
-- docs/advanced_setup.md
-- docs/command_line_interface.md
-- docs/error_handling.md
-- docs/inspecting_the_queue.md
-- docs/job_helper_methods.md
-- docs/logging.md
-- docs/managing_workers.md
-- docs/middleware.md
-- docs/migrating.md
-- docs/multiple_queues.md
-- docs/shutting_down_safely.md
-- docs/using_plain_connections.md
-- docs/using_sequel.md
-- docs/writing_reliable_jobs.md
 - lib/que.rb
 - lib/que/active_job/extensions.rb
 - lib/que/active_record/connection.rb
@@ -94,11 +79,11 @@ files:
 - lib/que/version.rb
 - lib/que/worker.rb
 - que.gemspec
-homepage: https://github.com/chanks/que
+homepage: https://github.com/que-rb/que
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -109,13 +94,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.3
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: A PostgreSQL-based Job Queue
 test_files: []