que 1.0.0.beta3 → 1.0.0.beta4

data/lib/que/active_record/connection.rb

@@ -17,7 +17,7 @@ module Que
 
         # Use Rails' executor (if present) to make sure that the connection
         # we're using isn't taken from us while the block runs. See
-        # https://github.com/chanks/que/issues/166#issuecomment-274218910
+        # https://github.com/que-rb/que/issues/166#issuecomment-274218910
         def wrap_in_rails_executor(&block)
           if defined?(::Rails.application.executor)
             ::Rails.application.executor.wrap(&block)

data/lib/que/connection.rb

@@ -152,7 +152,13 @@ module Que
       },
 
       # Timestamp with time zone
-      1184 => Time.method(:parse),
+      1184 => -> (value) {
+        case value
+        when Time then value
+        when String then Time.parse(value)
+        else raise "Unexpected time class: #{value.class} (#{value.inspect})"
+        end
+      }
     }
 
     # JSON, JSONB

data/lib/que/job_buffer.rb

@@ -107,10 +107,6 @@ module Que
       end
     end
 
-    def jobs_needed?
-      minimum_size > size
-    end
-
     def waiting_count
       count = 0
       priority_queues.each_value do |pq|

data/lib/que/locker.rb

@@ -301,12 +301,15 @@ module Que
 
     def poll
       # Only poll when there are pollers to use (that is, when polling is
-      # enabled) and when the local queue has dropped below the configured
-      # minimum size.
-      return unless pollers && job_buffer.jobs_needed?
+      # enabled).
+      return unless pollers
 
       # Figure out what job priorities we have to fill.
       priorities = job_buffer.available_priorities
+
+      # Only poll when there are workers ready for jobs.
+      return if priorities.empty?
+
       all_metajobs = []
 
       pollers.each do |poller|

data/lib/que/sequel/model.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+::Sequel.extension :pg_json_ops
+
 module Que
   module Sequel
     QUALIFIED_TABLE = ::Sequel.qualify(:public, :que_jobs)
@@ -7,10 +9,10 @@ module Que
     class Model < ::Sequel::Model(QUALIFIED_TABLE)
       dataset_module do
         conditions = {
-          errored:   ::Sequel.qualify(QUALIFIED_TABLE, :error_count) > 0,
-          expired:   ::Sequel.~(::Sequel.qualify(QUALIFIED_TABLE, :expired_at)  => nil),
-          finished:  ::Sequel.~(::Sequel.qualify(QUALIFIED_TABLE, :finished_at) => nil),
-          scheduled: ::Sequel.qualify(QUALIFIED_TABLE, :run_at) > ::Sequel::CURRENT_TIMESTAMP,
+          errored:   QUALIFIED_TABLE[:error_count] > 0,
+          expired:   QUALIFIED_TABLE[:expired_at] !~ nil,
+          finished:  QUALIFIED_TABLE[:finished_at] !~ nil,
+          scheduled: QUALIFIED_TABLE[:run_at] > ::Sequel::CURRENT_TIMESTAMP,
         }
 
         conditions.each do |name, condition|
@@ -18,32 +20,28 @@ module Que
           subset :"not_#{name}", ~condition
         end
 
-        subset :ready,     conditions.values.map(&:~).inject{|a, b| a & b}
-        subset :not_ready, conditions.values.         inject{|a, b| a | b}
+        subset :ready,     conditions.values.map(&:~).inject(:&)
+        subset :not_ready, conditions.values.         inject(:|)
 
         def by_job_class(job_class)
           job_class = job_class.name if job_class.is_a?(Class)
           where(
-            ::Sequel.|(
-              {::Sequel.qualify(QUALIFIED_TABLE, :job_class) => job_class},
-              {
-                ::Sequel.qualify(QUALIFIED_TABLE, :job_class) => "ActiveJob::QueueAdapters::QueAdapter::JobWrapper",
-                ::Sequel.lit("public.que_jobs.args->0->>'job_class'") => job_class,
-              }
-            )
+            (QUALIFIED_TABLE[:job_class] =~ job_class) |
+              (QUALIFIED_TABLE[:job_class] =~ "ActiveJob::QueueAdapters::QueAdapter::JobWrapper") &
+              (QUALIFIED_TABLE[:args].pg_jsonb[0].get_text("job_class") =~ job_class)
           )
         end
 
         def by_queue(queue)
-          where(::Sequel.qualify(QUALIFIED_TABLE, :queue) => queue)
+          where(QUALIFIED_TABLE[:queue] => queue)
         end
 
         def by_tag(tag)
-          where(::Sequel.lit("public.que_jobs.data @> ?", JSON.dump(tags: [tag])))
+          where(QUALIFIED_TABLE[:data].pg_jsonb.contains(JSON.dump(tags: [tag])))
         end
 
         def by_args(*args)
-          where(::Sequel.lit("public.que_jobs.args @> ?", JSON.dump(args)))
+          where(QUALIFIED_TABLE[:args].pg_jsonb.contains(JSON.dump(args)))
         end
       end
     end

data/lib/que/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Que
-  VERSION = '1.0.0.beta3'
+  VERSION = '1.0.0.beta4'
 end

data/que.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.email         = ['christopher.m.hanks@gmail.com']
   spec.description   = %q{A job queue that uses PostgreSQL's advisory locks for speed and reliability.}
   spec.summary       = %q{A PostgreSQL-based Job Queue}
-  spec.homepage      = 'https://github.com/chanks/que'
+  spec.homepage      = 'https://github.com/que-rb/que'
   spec.license       = 'MIT'
 
   files_to_exclude = [
@@ -29,5 +29,5 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'bundler'
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: que
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta3
+  version: 1.0.0.beta4
 platform: ruby
 authors:
 - Chris Hanks
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-18 00:00:00.000000000 Z
+date: 2020-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
 description: A job queue that uses PostgreSQL's advisory locks for speed and reliability.
 email:
 - christopher.m.hanks@gmail.com
@@ -32,6 +32,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - CHANGELOG.1.0.beta.md
 - CHANGELOG.md
@@ -41,21 +42,6 @@ files:
 - bin/command_line_interface.rb
 - bin/que
 - docs/README.md
-- docs/active_job.md
-- docs/advanced_setup.md
-- docs/command_line_interface.md
-- docs/error_handling.md
-- docs/inspecting_the_queue.md
-- docs/job_helper_methods.md
-- docs/logging.md
-- docs/managing_workers.md
-- docs/middleware.md
-- docs/migrating.md
-- docs/multiple_queues.md
-- docs/shutting_down_safely.md
-- docs/using_plain_connections.md
-- docs/using_sequel.md
-- docs/writing_reliable_jobs.md
 - lib/que.rb
 - lib/que/active_job/extensions.rb
 - lib/que/active_record/connection.rb
@@ -94,7 +80,7 @@ files:
 - lib/que/version.rb
 - lib/que/worker.rb
 - que.gemspec
-homepage: https://github.com/chanks/que
+homepage: https://github.com/que-rb/que
 licenses:
 - MIT
 metadata: {}
@@ -113,8 +99,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: A PostgreSQL-based Job Queue

data/docs/active_job.md

@@ -1,6 +0,0 @@
-## Using Que With ActiveJob
-
-You can include `Que::ActiveJob::JobExtensions` into your `ApplicationJob` subclass to get support for all of Que's 
-[helper methods](/docs/job_helper_methods.md). These methods will become no-ops if you use a queue adapter that isn't Que, so if you like to use a different adapter in development they shouldn't interfere.
-
-Additionally, including `Que::ActiveJob::JobExtensions` lets you define a run() method that supports keyword arguments.

data/docs/advanced_setup.md

@@ -1,49 +0,0 @@
-## Advanced Setup
-
-### Using ActiveRecord Without Rails
-
-If you're using both Rails and ActiveRecord, the README describes how to get started with Que (which is pretty straightforward, since it includes a Railtie that handles a lot of setup for you). Otherwise, you'll need to do some manual setup.
-
-If you're using ActiveRecord outside of Rails, you'll need to tell Que to piggyback on its connection pool after you've connected to the database:
-
-```ruby
-ActiveRecord::Base.establish_connection(ENV['DATABASE_URL'])
-
-require 'que'
-Que.connection = ActiveRecord
-```
-
-Then you can queue jobs just as you would in Rails:
-
-```ruby
-ActiveRecord::Base.transaction do
-  @user = User.create(params[:user])
-  SendRegistrationEmail.enqueue user_id: @user.id
-end
-```
-
-There are other docs to read if you're using [Sequel](https://github.com/chanks/que/blob/master/docs/using_sequel.md) or [plain Postgres connections](https://github.com/chanks/que/blob/master/docs/using_plain_connections.md) (with no ORM at all) instead of ActiveRecord.
-
-### Managing the Jobs Table
-
-After you've connected Que to the database, you can manage the jobs table. You'll want to migrate to a specific version in a migration file, to ensure that they work the same way even when you upgrade Que in the future:
-
-```ruby
-# Update the schema to version #4.
-Que.migrate! version: 4
-
-# Remove Que's jobs table entirely.
-Que.migrate! version: 0
-```
-
-There's also a helper method to clear all jobs from the jobs table:
-
-```ruby
-Que.clear!
-```
-
-### Other Setup
-
-Be sure to read the docs on [managing workers](https://github.com/chanks/que/blob/master/docs/managing_workers.md) for more information on using the worker pool.
-
-You'll also want to set up [logging](https://github.com/chanks/que/blob/master/docs/logging.md) and an [error handler](https://github.com/chanks/que/blob/master/docs/error_handling.md) to track errors raised by jobs.

data/docs/command_line_interface.md

@@ -1,49 +0,0 @@
-## Command Line Interface
-
-```
-usage: que [options] [file/to/require] ...
-    -h, --help                       Show this help text.
-    -i, --poll-interval [INTERVAL]   Set maximum interval between polls for available jobs, in seconds (default: 5)
-    -l, --log-level [LEVEL]          Set level at which to log to STDOUT (debug, info, warn, error, fatal) (default: info)
-    -p, --worker-priorities [LIST]   List of priorities to assign to workers (default: 10,30,50,any,any,any)
-    -q, --queue-name [NAME]          Set a queue name to work jobs from. Can be passed multiple times. (default: the default queue only)
-    -w, --worker-count [COUNT]       Set number of workers in process (default: 6)
-    -v, --version                    Print Que version and exit.
-        --connection-url [URL]       Set a custom database url to connect to for locking purposes.
-        --log-internals              Log verbosely about Que's internal state. Only recommended for debugging issues
-        --maximum-buffer-size [SIZE] Set maximum number of jobs to be locked and held in this process awaiting a worker (default: 8)
-        --minimum-buffer-size [SIZE] Set minimum number of jobs to be locked and held in this process awaiting a worker (default: 2)
-        --wait-period [PERIOD]       Set maximum interval between checks of the in-memory job queue, in milliseconds (default: 50)
-```
-
-Some explanation of the more unusual options:
-
-### worker-priorities and worker-count
-
-These options dictate the size and priority distribution of the worker pool. The default worker-priorities is `10,30,50,any,any,any`. This means that the default worker pool will reserve one worker to only works jobs with priorities under 10, one for priorities under 30, and one for priorities under 50. Three more workers will work any job.
-
-For example, with these defaults, you could have a large backlog of jobs of priority 100. When a more important job (priority 40) comes in, there's guaranteed to be a free worker. If the process then becomes saturated with jobs of priority 40, and then a priority 20 job comes in, there's guaranteed to be a free worker for it, and so on. You can pass a priority more than once to have multiple workers at that level (for example: `--worker-priorities=100,100,any,any`). This gives you a lot of freedom to manage your worker capacity at different priority levels.
-
-Instead of passing worker-priorities, you can pass a `worker-count` - this is a shorthand for creating the given number of workers at the `any` priority level. So, `--worker-count=3` is just like passing equivalent to `worker-priorities=any,any,any`.
-
-If you pass both worker-count and worker-priorities, the count will trim or pad the priorities list with `any` workers. So, `--worker-priorities=20,30,40 --worker-count=6` would be the same as passing `--worker-priorities=20,30,40,any,any,any`.
-
-### poll-interval
-
-This option sets the number of seconds the process will wait between polls of the job queue. Jobs that are ready to be worked immediately will be broadcast via the LISTEN/NOTIFY system, so polling is unnecessary for them - polling is only necessary for jobs that are scheduled in the future or which are being delayed due to errors. The default is 5 seconds.
-
-### minimum-buffer-size and maximum-buffer-size
-
-These options set the size of the internal buffer that Que uses to hold jobs until they're ready for workers. The default minimum is 2 and the maximum is 8, meaning that the process won't buffer more than 8 jobs that aren't yet ready to be worked, and will only resort to polling if the buffer dips below 2. If you don't want jobs to be buffered at all, you can set both of these values to zero.
-
-### connection-url
-
-This option sets the URL to be used to open a connection to the database for locking purposes. By default, Que will simply use a connection from the connection pool for locking - this option is only useful if your application connections can't use advisory locks - for example, if they're passed through an external connection pool like PgBouncer. In that case, you'll need to use this option to specify your actual database URL so that Que can establish a direct connection.
-
-### wait-period
-
-This option specifies (in milliseconds) how often the locking thread wakes up to check whether the workers have finished jobs, whether it's time to poll, etc. You shouldn't generally need to tweak this, but it may come in handy for some workloads. The default is 50 milliseconds.
-
-### log-internals
-
-This option instructs Que to output a lot of information about its internal state to the logger. It should only be used if it becomes necessary to debug issues.

data/docs/error_handling.md

@@ -1,94 +0,0 @@
-## Error Handling
-
-If an error is raised and left uncaught by your job, Que will save the error message and backtrace to the database and schedule the job to be retried later.
-
-If a given job fails repeatedly, Que will retry it at exponentially-increasing intervals equal to (failure_count^4 + 3) seconds. This means that a job will be retried 4 seconds after its first failure, 19 seconds after its second, 84 seconds after its third, 259 seconds after its fourth, and so on until it succeeds. This pattern is very similar to DelayedJob's. Alternately, you can define your own retry logic by setting an interval to delay each time, or a callable that accepts the number of failures and returns an interval:
-
-```ruby
-class MyJob < Que::Job
-  # Just retry a failed job every 5 seconds:
-  self.retry_interval = 5
-
-  # Always retry this job immediately (not recommended, or transient
-  # errors will spam your error reporting):
-  self.retry_interval = 0
-
-  # Increase the delay by 30 seconds every time this job fails:
-  self.retry_interval = proc { |count| count * 30 }
-end
-```
-
-There is a maximum_retry_count option for jobs. It defaults to 15 retries, which with the default retry interval means that a job will stop retrying after a little more than two days.
-
-## Error Notifications
-
-If you're using an error notification system (highly recommended, of course), you can hook Que into it by setting a callable as the error notifier:
-
-```ruby
-Que.error_notifier = proc do |error, job|
-  # Do whatever you want with the error object or job row here. Note that the
-  # job passed is not the actual job object, but the hash representing the job
-  # row in the database, which looks like:
-
-  # {
-  #   :priority => 100,
-  #   :run_at => "2017-09-15T20:18:52.018101Z",
-  #   :id => 172340879,
-  #   :job_class => "TestJob",
-  #   :error_count => 0,
-  #   :last_error_message => nil,
-  #   :queue => "default",
-  #   :last_error_backtrace => nil,
-  #   :finished_at => nil,
-  #   :expired_at => nil,
-  #   :args => [],
-  #   :data => {}
-  # }
-
-  # This is done because the job may not have been able to be deserialized
-  # properly, if the name of the job class was changed or the job class isn't
-  # loaded for some reason. The job argument may also be nil, if there was a
-  # connection failure or something similar.
-end
-```
-
-## Error-Specific Handling
-
-You can also define a handle_error method in your job, like so:
-
-```ruby
-class MyJob < Que::Job
-  def run(*args)
-    # Your code goes here.
-  end
-
-  def handle_error(error)
-    case error
-    when TemporaryError then retry_in 10.seconds
-    when PermanentError then expire
-    else super # Default (exponential backoff) behavior.
-    end
-  end
-end
-```
-
-The return value of handle_error determines whether the error object is passed to the error notifier. The helper methods like expire and retry_in return true, so these errors will be notified. You can explicitly return false to skip notification.
-
-```ruby
-class MyJob < Que::Job
-  def handle_error(error)
-    case error
-    when AnnoyingError
-      retry_in 10.seconds
-      false
-    when TransientError
-      super
-      error_count > 3
-    else
-      super # Default (exponential backoff) behavior.
-    end
-  end
-end
-```
-
-In this example, AnnoyingError will never be notified, while TransientError will only be notified once it has affected a given job at least three times.

data/docs/inspecting_the_queue.md

@@ -1,64 +0,0 @@
-## Inspecting the Queue
-
-In order to remain simple and compatible with any ORM (or no ORM at all), Que is really just a very thin wrapper around some raw SQL. There are two methods available that query the jobs table and Postgres' system catalogs to retrieve information on the current state of the queue:
-
-### Job Stats
-
-You can call `Que.job_stats` to return some aggregate data on the types of jobs currently in the queue. Example output:
-
-```ruby
-[
-  {
-    :job_class=>"ChargeCreditCard",
-    :count=>10,
-    :count_working=>4,
-    :count_errored=>2,
-    :highest_error_count=>5,
-    :oldest_run_at=>2017-09-08 16:13:18 -0400
-  },
-  {
-    :job_class=>"SendRegistrationEmail",
-    :count=>1,
-    :count_working=>0,
-    :count_errored=>0,
-    :highest_error_count=>0,
-    :oldest_run_at=>2017-09-08 17:13:18 -0400
-  }
-]
-```
-
-This tells you that, for instance, there are ten ChargeCreditCard jobs in the queue, four of which are currently being worked, and two of which have experienced errors. One of them has started to process but experienced an error five times. The oldest_run_at is helpful for determining how long jobs have been sitting around, if you have a large backlog.
-
-### Custom Queries
-
-If you're using ActiveRecord or Sequel, Que ships with models that wrap the job queue so you can write your own logic to inspect it. They include some helpful scopes to write your queries - see the gem source for a complete accounting.
-
-#### ActiveRecord Example
-
-``` ruby
-# app/models/que_job.rb
-
-require 'que/active_record/model'
-
-class QueJob < Que::ActiveRecord::Model
-end
-
-QueJob.finished.to_sql # => "SELECT \"que_jobs\".* FROM \"que_jobs\" WHERE (\"que_jobs\".\"finished_at\" IS NOT NULL)"
-
-# You could also name the model whatever you like, or just query from
-# Que::ActiveRecord::Model directly if you don't need to write your own model
-# logic.
-```
-
-#### Sequel Example
-
-``` ruby
-# app/models/que_job.rb
-
-require 'que/sequel/model'
-
-class QueJob < Que::Sequel::Model
-end
-
-QueJob.finished # => #<Sequel::Postgres::Dataset: "SELECT * FROM \"public\".\"que_jobs\" WHERE (\"public\".\"que_jobs\".\"finished_at\" IS NOT NULL)">
-```