que 1.0.0.beta3 → 1.1.0

data/lib/que/active_record/connection.rb

@@ -17,7 +17,7 @@ module Que
 
         # Use Rails' executor (if present) to make sure that the connection
         # we're using isn't taken from us while the block runs. See
-        # https://github.com/chanks/que/issues/166#issuecomment-274218910
+        # https://github.com/que-rb/que/issues/166#issuecomment-274218910
         def wrap_in_rails_executor(&block)
           if defined?(::Rails.application.executor)
             ::Rails.application.executor.wrap(&block)

data/lib/que/active_record/model.rb

@@ -3,7 +3,7 @@
 module Que
   module ActiveRecord
     class Model < ::ActiveRecord::Base
-      self.table_name = :que_jobs
+      self.table_name = 'public.que_jobs'
 
       t = arel_table
 

data/lib/que/connection.rb

@@ -119,6 +119,10 @@ module Que
       loop { break if next_notification.nil? }
     end
 
+    def server_version
+      wrapped_connection.server_version
+    end
+
     def in_transaction?
       wrapped_connection.transaction_status != ::PG::PQTRANS_IDLE
     end
@@ -152,7 +156,13 @@ module Que
       },
 
       # Timestamp with time zone
-      1184 => Time.method(:parse),
+      1184 => -> (value) {
+        case value
+        when Time then value
+        when String then Time.parse(value)
+        else raise "Unexpected time class: #{value.class} (#{value.inspect})"
+        end
+      }
     }
 
     # JSON, JSONB

data/lib/que/job_buffer.rb

@@ -59,10 +59,8 @@ module Que
 
         # Relying on the hash's contents being sorted, here.
         priority_queues.reverse_each do |_, pq|
-          pq.waiting_count.times do
-            job = _shift_job(pq.priority)
-            break if job.nil? # False would mean we're stopping.
-            pq.push(job)
+          pq.populate do
+            _shift_job(pq.priority)
           end
         end
 
@@ -75,7 +73,7 @@ module Que
 
     def shift(priority = nil)
       queue = priority_queues.fetch(priority) { raise Error, "not a permitted priority! #{priority}" }
-      queue.pop
+      queue.pop || shift_job(priority)
     end
 
     def shift_job(priority = nil)
@@ -107,10 +105,6 @@ module Que
       end
     end
 
-    def jobs_needed?
-      minimum_size > size
-    end
-
     def waiting_count
       count = 0
       priority_queues.each_value do |pq|
@@ -162,6 +156,10 @@ module Que
       sync { _stopping? }
     end
 
+    def job_available?(priority)
+      (job = @array.first) && job.priority_sufficient?(priority)
+    end
+
     private
 
     def _buffer_space
@@ -214,15 +212,14 @@ module Que
       def pop
         sync do
           loop do
-            return false if @stopping
-
-            if item = @items.pop
+            if @stopping
+              return false
+            elsif item = @items.pop
               return item
+            elsif job_buffer.job_available?(priority)
+              return false
             end
 
-            job = job_buffer.shift_job(priority)
-            return job unless job.nil? # False means we're stopping.
-
             @waiting += 1
             @cv.wait(mutex)
             @waiting -= 1
@@ -230,18 +227,20 @@ module Que
         end
       end
 
-      def push(item)
+      def stop
         sync do
-          Que.assert(waiting_count > 0)
-          @items << item
-          @cv.signal
+          @stopping = true
+          @cv.broadcast
         end
       end
 
-      def stop
+      def populate
         sync do
-          @stopping = true
-          @cv.broadcast
+          waiting_count.times do
+            job = yield
+            break if job.nil? # False would mean we're stopping.
+            _push(job)
+          end
         end
       end
 
@@ -254,6 +253,12 @@ module Que
       def sync(&block)
         mutex.synchronize(&block)
       end
+
+      def _push(item)
+        Que.assert(waiting_count > 0)
+        @items << item
+        @cv.signal
+      end
     end
   end
 end

data/lib/que/locker.rb

@@ -301,12 +301,15 @@ module Que
 
     def poll
       # Only poll when there are pollers to use (that is, when polling is
-      # enabled) and when the local queue has dropped below the configured
-      # minimum size.
-      return unless pollers && job_buffer.jobs_needed?
+      # enabled).
+      return unless pollers
 
       # Figure out what job priorities we have to fill.
       priorities = job_buffer.available_priorities
+
+      # Only poll when there are workers ready for jobs.
+      return if priorities.empty?
+
       all_metajobs = []
 
       pollers.each do |poller|
@@ -390,10 +393,12 @@ module Que
         }
       end
 
+      materalize_cte = connection.server_version >= 12_00_00
+
       jobs =
         connection.execute \
           <<-SQL
-            WITH jobs AS (SELECT * FROM que_jobs WHERE id IN (#{ids.join(', ')}))
+            WITH jobs AS #{materalize_cte ? 'MATERIALIZED' : ''} (SELECT * FROM que_jobs WHERE id IN (#{ids.join(', ')}))
             SELECT * FROM jobs WHERE pg_try_advisory_lock(id)
           SQL
 

data/lib/que/sequel/model.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+::Sequel.extension :pg_json_ops
+
 module Que
   module Sequel
     QUALIFIED_TABLE = ::Sequel.qualify(:public, :que_jobs)
@@ -7,10 +9,10 @@ module Que
     class Model < ::Sequel::Model(QUALIFIED_TABLE)
       dataset_module do
         conditions = {
-          errored:   ::Sequel.qualify(QUALIFIED_TABLE, :error_count) > 0,
-          expired:   ::Sequel.~(::Sequel.qualify(QUALIFIED_TABLE, :expired_at)  => nil),
-          finished:  ::Sequel.~(::Sequel.qualify(QUALIFIED_TABLE, :finished_at) => nil),
-          scheduled: ::Sequel.qualify(QUALIFIED_TABLE, :run_at) > ::Sequel::CURRENT_TIMESTAMP,
+          errored:   QUALIFIED_TABLE[:error_count] > 0,
+          expired:   QUALIFIED_TABLE[:expired_at] !~ nil,
+          finished:  QUALIFIED_TABLE[:finished_at] !~ nil,
+          scheduled: QUALIFIED_TABLE[:run_at] > ::Sequel::CURRENT_TIMESTAMP,
         }
 
         conditions.each do |name, condition|
@@ -18,32 +20,28 @@ module Que
           subset :"not_#{name}", ~condition
         end
 
-        subset :ready,     conditions.values.map(&:~).inject{|a, b| a & b}
-        subset :not_ready, conditions.values.         inject{|a, b| a | b}
+        subset :ready,     conditions.values.map(&:~).inject(:&)
+        subset :not_ready, conditions.values.         inject(:|)
 
         def by_job_class(job_class)
           job_class = job_class.name if job_class.is_a?(Class)
           where(
-            ::Sequel.|(
-              {::Sequel.qualify(QUALIFIED_TABLE, :job_class) => job_class},
-              {
-                ::Sequel.qualify(QUALIFIED_TABLE, :job_class) => "ActiveJob::QueueAdapters::QueAdapter::JobWrapper",
-                ::Sequel.lit("public.que_jobs.args->0->>'job_class'") => job_class,
-              }
-            )
+            (QUALIFIED_TABLE[:job_class] =~ job_class) |
+              (QUALIFIED_TABLE[:job_class] =~ "ActiveJob::QueueAdapters::QueAdapter::JobWrapper") &
+              (QUALIFIED_TABLE[:args].pg_jsonb[0].get_text("job_class") =~ job_class)
           )
         end
 
         def by_queue(queue)
-          where(::Sequel.qualify(QUALIFIED_TABLE, :queue) => queue)
+          where(QUALIFIED_TABLE[:queue] => queue)
         end
 
         def by_tag(tag)
-          where(::Sequel.lit("public.que_jobs.data @> ?", JSON.dump(tags: [tag])))
+          where(QUALIFIED_TABLE[:data].pg_jsonb.contains(JSON.dump(tags: [tag])))
         end
 
         def by_args(*args)
-          where(::Sequel.lit("public.que_jobs.args @> ?", JSON.dump(args)))
+          where(QUALIFIED_TABLE[:args].pg_jsonb.contains(JSON.dump(args)))
         end
       end
     end

data/lib/que/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Que
-  VERSION = '1.0.0.beta3'
+  VERSION = '1.1.0'
 end

data/lib/que/worker.rb

@@ -54,10 +54,17 @@ module Que
     private
 
     def work_loop
-      # Blocks until a job of the appropriate priority is available. If the
-      # queue is shutting down this will return nil, which breaks the loop and
+      # Blocks until a job of the appropriate priority is available.
+      # `fetch_next_metajob` normally returns a job to be processed.
+      # If the queue is shutting down it will return false, which breaks the loop and
       # lets the thread finish.
-      while metajob = fetch_next_metajob
+      while (metajob = fetch_next_metajob) != false
+        # If metajob is nil instead of false, we've hit a rare race condition where
+        # there was a job in the buffer when the worker code checked, but the job was
+        # picked up by the time we got around to shifting it off the buffer.
+        # Letting this case go unhandled leads to worker threads exiting pre-maturely, so
+        # we check explicitly and continue the loop.
+        next if metajob.nil?
         id = metajob.id
 
         Que.internal_log(:worker_received_job, self) { {id: id} }
@@ -130,6 +137,7 @@ module Que
         error: {
           class:   error.class.to_s,
           message: error.message,
+          backtrace: (error.backtrace || []).join("\n").slice(0, 10000),
         },
       )
 
@@ -157,7 +165,7 @@ module Que
           Que.execute :set_error, [
             delay,
             "#{error.class}: #{error.message}".slice(0, 500),
-            error.backtrace.join("\n").slice(0, 10000),
+            (error.backtrace || []).join("\n").slice(0, 10000),
             job.fetch(:id),
           ]
         end

data/que.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.email         = ['christopher.m.hanks@gmail.com']
   spec.description   = %q{A job queue that uses PostgreSQL's advisory locks for speed and reliability.}
   spec.summary       = %q{A PostgreSQL-based Job Queue}
-  spec.homepage      = 'https://github.com/chanks/que'
+  spec.homepage      = 'https://github.com/que-rb/que'
   spec.license       = 'MIT'
 
   files_to_exclude = [
@@ -29,5 +29,5 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'bundler'
 end

data/scripts/docker-entrypoint

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+# For using your own dotfiles within the Docker container
+if [ -f /.docker-rc.d/.docker-bashrc ]; then
+  echo "source /.docker-rc.d/.docker-bashrc" >> ~/.bashrc
+fi
+
+gem list -i -e bundler -v "$RUBY_BUNDLER_VERSION" >/dev/null || gem install bundler -v "$RUBY_BUNDLER_VERSION"
+
+bundle check --dry-run || bundle install
+
+exec "${@-bash}"

data/scripts/test

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+bundle exec rake spec "$@"

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: que
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta3
+  version: 1.1.0
 platform: ruby
 authors:
 - Chris Hanks
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-18 00:00:00.000000000 Z
+date: 2022-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
 description: A job queue that uses PostgreSQL's advisory locks for speed and reliability.
 email:
 - christopher.m.hanks@gmail.com
@@ -32,30 +32,21 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/tests.yml"
 - ".gitignore"
-- CHANGELOG.1.0.beta.md
 - CHANGELOG.md
+- Dockerfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- auto/dev
+- auto/psql
+- auto/test
+- auto/test-postgres-14
 - bin/command_line_interface.rb
 - bin/que
+- docker-compose.yml
 - docs/README.md
-- docs/active_job.md
-- docs/advanced_setup.md
-- docs/command_line_interface.md
-- docs/error_handling.md
-- docs/inspecting_the_queue.md
-- docs/job_helper_methods.md
-- docs/logging.md
-- docs/managing_workers.md
-- docs/middleware.md
-- docs/migrating.md
-- docs/multiple_queues.md
-- docs/shutting_down_safely.md
-- docs/using_plain_connections.md
-- docs/using_sequel.md
-- docs/writing_reliable_jobs.md
 - lib/que.rb
 - lib/que/active_job/extensions.rb
 - lib/que/active_record/connection.rb
@@ -94,11 +85,13 @@ files:
 - lib/que/version.rb
 - lib/que/worker.rb
 - que.gemspec
-homepage: https://github.com/chanks/que
+- scripts/docker-entrypoint
+- scripts/test
+homepage: https://github.com/que-rb/que
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -109,13 +102,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.3
-signing_key: 
+rubygems_version: 3.3.6
+signing_key:
 specification_version: 4
 summary: A PostgreSQL-based Job Queue
 test_files: []

data/CHANGELOG.1.0.beta.md

@@ -1,137 +0,0 @@
-### 1.0.0.beta3 (2018-05-18)
-
-*   Added support for customizing log levels for `job_worked` events (#217).
-
-*   Began logging all `job_errored` events at the `ERROR` log level.
-
-*   Fixed the Railtie when running in test mode (#214).
-
-*   Tweaked the meanings of worker-priorities and worker-count options in the CLI, to better support use cases with low worker counts (#216).
-
-### 1.0.0.beta2 (2018-04-13)
-
-*   Fixed an incompatibility that caused the new locker to hang when using Rails in development mode (#213).
-
-*   Fixed a bug with setting the log level via the CLI when the configured logger was based on a callable (#210).
-
-*   Renamed Que.middleware to Que.job_middleware.
-
-*   Added Que.sql_middleware.
-
-*   Officially added support for Ruby 2.5.
-
-*   Internal cleanup and renamings.
-
-### 1.0.0.beta (2017-10-25)
-
-*   **A schema upgrade to version 4 will be required for this release.** See [the migration doc](https://github.com/chanks/que/blob/master/docs/migrating.md) for information if you're upgrading from a previous release.
-
-    *   Please note that this migration requires a rewrite of the jobs table, which makes it O(n) with the size of the table. If you have a very large backlog of jobs you may want to schedule downtime for this migration.
-
-*   Que's implementation has been changed from one in which worker threads hold their own PG connections and lock their own jobs to one in which a single thread (and PG connection) locks jobs through LISTEN/NOTIFY and batch polling, and passes jobs along to worker threads. This has many benefits, including:
-
-    *   Jobs queued for immediate processing can be actively distributed to workers with LISTEN/NOTIFY, which is more efficient than having workers repeatedly poll for new jobs.
-
-    *   When polling is necessary (to pick up jobs that are scheduled for the future or that need to be retried due to errors), jobs can be locked and fetched in batches, rather than one at a time.
-
-    *   Individual workers no longer need to monopolize their own (usually idle) connections while working jobs, so Ruby processes will require fewer Postgres connections.
-
-    *   PgBouncer or another external connection pool can be used for workers' connections (though not for the connection used to lock and listen for jobs).
-
-*   Other features introduced in this version include:
-
-    *   Much better support for all versions of ActiveJob.
-
-        *   In particular, you may (optionally) include `Que::ActiveJob::JobExtensions` into `ApplicationJob` to get support for all of Que's job helper methods.
-
-    *   Custom middleware that wrap running jobs are now supported.
-
-    *   Support for categorizing jobs with tags.
-
-    *   Support for configuring a `maximum_retry_count` on individual job classes.
-
-    *   Job configuration options are now inheritable, so job class hierarchies are more useful.
-
-    *   There are now built-in models for ActiveRecord and Sequel to allow inspecting the queue easily.
-
-    *   Jobs that have finished working may optionally be retained in the database indefinitely.
-
-        *   To keep a job record, replace the `destroy` calls in your jobs with `finish`. `destroy` will still delete records entirely, for jobs that you don't want to keep.
-
-        *   If you don't resolve a job yourself one way or another, Que will still `destroy` the job for you by default.
-
-        *   Finished jobs have a timestamp set in the finished_at column.
-
-    *   Jobs that have errored too many times will now be marked as expired, and won't be retried again.
-
-        *   You can configure a maximum_retry_count in your job classes, to set the threshold at which a job will be marked expired. The default is 15.
-
-        *   To manually mark a job as expired (and keep it in the database but not try to run it again) you can call `expire` helper in your job.
-
-    *   You can now set job priority thresholds for individual workers, to ensure that there will always be space available for high-priority jobs.
-
-    *   `Que.job_states` returns a list of locked jobs and the hostname/pid of the Ruby processes that have locked them.
-
-    *   `Que.connection_proc=` has been added, to allow for the easy integration of custom connection pools.
-
-*   In keeping with semantic versioning, the major version is being bumped since the new implementation requires some backwards-incompatible changes. These changes include:
-
-    *   Support for MRI Rubies before 2.2 has been dropped.
-
-    *   Support for Postgres versions before 9.5 has been dropped (JSONB and upsert support is required).
-
-    *   JRuby support has been dropped. It will be reintroduced whenever the jruby-pg gem is production-ready.
-
-    *   The `que:work` rake task has been removed. Use the `que` executable instead.
-
-        *   Therefore, configuring workers using QUE_* environment variables is no longer supported. Please pass the appropriate options to the `que` executable instead.
-
-    *   The `mode` setter has been removed.
-
-        *   To run jobs synchronously when they are enqueued (the old `:sync` behavior) you can set `Que.run_synchronously = true`.
-
-        *   To start up the worker pool (the old :async behavior) you should use the `que` executable to start up a worker process. There's no longer a supported API for running workers outside of the `que` executable.
-
-    *   The following methods are not meaningful under the new implementation and have been removed:
-
-        *   The `Que.wake_interval` getter and setter.
-
-        *   The `Que.worker_count` getter and setter.
-
-        *   `Que.wake!`
-
-        *   `Que.wake_all!`
-
-    *   Since Que needs a dedicated Postgres connection to manage job locks, running Que through a single PG connection is no longer supported.
-
-        *   It's not clear that anyone ever actually did this.
-
-    *   `Que.worker_states` has been removed, as the connection that locks a job is no longer the one that the job is using to run. Its functionality has been partially replaced with `Que.job_states`.
-
-    *   When using Rails, for simplicity, job attributes and keys in argument hashes are now converted to symbols when retrieved from the database, rather than being converted to instances of HashWithIndifferentAccess.
-
-    *   Arguments passed to jobs are now deep-frozen, to prevent unexpected behavior when the args are mutated and the job is reenqueued.
-
-    *   Since JSONB is now used to store arguments, the order of argument hashes is no longer maintained.
-
-        *   It wouldn't have been a good idea to rely on this anyway.
-
-    *   Calling Que.log() directly is no longer supported/recommended.
-
-    *   Features marked as deprecated in the final 0.x releases have been removed.
-
-*   Finally, if you've built up your own tooling and customizations around Que, you may need to be aware of some DB schema changes made in the migration to schema version #4.
-
-    *   The `job_id` column has been renamed `id` and is now the primary key. This makes it easier to manage the queue using an ActiveRecord model.
-
-    *   Finished jobs are now kept in the DB, unless you explicitly call `destroy`. If you want to query the DB for only jobs that haven't finished yet, add a `WHERE finished_at IS NULL` condition to your query, or use the not_finished scope on one of the provided ORM models.
-
-    *   There is now an `expired_at` timestamp column, which is set when a job reaches its maximum number of retries and will not be attempted again.
-
-    *   Due to popular demand, the default queue name is now "default" rather than an empty string. The migration will move pending jobs under the "" queue to the "default" queue.
-
-    *   The `last_error` column has been split in two, to `last_error_message` and `last_error_backtrace`. These two columns are now limited to 500 and 10,000 characters, respectively. The migration will split old error data correctly, and truncate it if necessary.
-
-    *   Names for queues and job classes are now limited to 500 characters, which is still far longer than either of these values should reasonably be.
-
-    *   There is now a `data` JSONB column which is used to support various ways of organizing jobs (setting tags on them, etc).

data/docs/active_job.md

@@ -1,6 +0,0 @@
-## Using Que With ActiveJob
-
-You can include `Que::ActiveJob::JobExtensions` into your `ApplicationJob` subclass to get support for all of Que's 
-[helper methods](/docs/job_helper_methods.md). These methods will become no-ops if you use a queue adapter that isn't Que, so if you like to use a different adapter in development they shouldn't interfere.
-
-Additionally, including `Que::ActiveJob::JobExtensions` lets you define a run() method that supports keyword arguments.

data/docs/advanced_setup.md

@@ -1,49 +0,0 @@
-## Advanced Setup
-
-### Using ActiveRecord Without Rails
-
-If you're using both Rails and ActiveRecord, the README describes how to get started with Que (which is pretty straightforward, since it includes a Railtie that handles a lot of setup for you). Otherwise, you'll need to do some manual setup.
-
-If you're using ActiveRecord outside of Rails, you'll need to tell Que to piggyback on its connection pool after you've connected to the database:
-
-```ruby
-ActiveRecord::Base.establish_connection(ENV['DATABASE_URL'])
-
-require 'que'
-Que.connection = ActiveRecord
-```
-
-Then you can queue jobs just as you would in Rails:
-
-```ruby
-ActiveRecord::Base.transaction do
-  @user = User.create(params[:user])
-  SendRegistrationEmail.enqueue user_id: @user.id
-end
-```
-
-There are other docs to read if you're using [Sequel](https://github.com/chanks/que/blob/master/docs/using_sequel.md) or [plain Postgres connections](https://github.com/chanks/que/blob/master/docs/using_plain_connections.md) (with no ORM at all) instead of ActiveRecord.
-
-### Managing the Jobs Table
-
-After you've connected Que to the database, you can manage the jobs table. You'll want to migrate to a specific version in a migration file, to ensure that they work the same way even when you upgrade Que in the future:
-
-```ruby
-# Update the schema to version #4.
-Que.migrate! version: 4
-
-# Remove Que's jobs table entirely.
-Que.migrate! version: 0
-```
-
-There's also a helper method to clear all jobs from the jobs table:
-
-```ruby
-Que.clear!
-```
-
-### Other Setup
-
-Be sure to read the docs on [managing workers](https://github.com/chanks/que/blob/master/docs/managing_workers.md) for more information on using the worker pool.
-
-You'll also want to set up [logging](https://github.com/chanks/que/blob/master/docs/logging.md) and an [error handler](https://github.com/chanks/que/blob/master/docs/error_handling.md) to track errors raised by jobs.