que 0.14.3 → 1.0.0.beta

data/bin/que

@@ -1,88 +1,14 @@
 #!/usr/bin/env ruby
+# frozen_string_literal: true
 
-require 'optparse'
-require 'ostruct'
-require 'logger'
+require_relative 'command_line_interface'
 
 $stdout.sync = true
 
-options = OpenStruct.new
+exit_code =
+  Que::CommandLineInterface.parse(
+    args:   ARGV.dup,
+    output: $stdout,
+  )
 
-OptionParser.new do |opts|
-  opts.banner = 'usage: que [options] file/to/require ...'
-
-  opts.on('-w', '--worker-count [COUNT]', Integer, "Set number of workers in process (default: 4)") do |worker_count|
-    options.worker_count = worker_count
-  end
-
-  opts.on('-i', '--wake-interval [INTERVAL]', Float, "Set maximum interval between polls of the job queue (in seconds) (default: 0.1)") do |wake_interval|
-    options.wake_interval = wake_interval
-  end
-
-  opts.on('-l', '--log-level [LEVEL]', String, "Set level of Que's logger (debug, info, warn, error, fatal) (default: info)") do |log_level|
-    options.log_level = log_level
-  end
-
-  opts.on('-q', '--queue-name [NAME]', String, "Set the name of the queue to work jobs from (default: the default queue)") do |queue_name|
-    options.queue_name = queue_name
-  end
-
-  opts.on('-v', '--version', "Show Que version") do
-    require 'que'
-    $stdout.puts "Que version #{Que::Version}"
-    exit 0
-  end
-
-  opts.on('-h', '--help', "Show help text") do
-    $stdout.puts opts
-    exit 0
-  end
-end.parse!(ARGV)
-
-if ARGV.length.zero?
-  $stdout.puts <<-OUTPUT
-You didn't include any Ruby files to require!
-Que needs to be able to load your application before it can process jobs.
-(Hint: If you're using Rails, try `que ./config/environment.rb`)
-(Or use `que -h` for a list of options)
-OUTPUT
-exit 1
-end
-
-ARGV.each do |file|
-  begin
-    require file
-  rescue LoadError
-    $stdout.puts "Could not load file '#{file}'"
-  end
-end
-
-Que.logger ||= Logger.new(STDOUT)
-
-begin
-  if log_level = (options.log_level || ENV['QUE_LOG_LEVEL'])
-    Que.logger.level = Logger.const_get(log_level.upcase)
-  end
-rescue NameError
-  $stdout.puts "Bad logging level: #{log_level}"
-  exit 1
-end
-
-Que.queue_name    = options.queue_name     || ENV['QUE_QUEUE']         || Que.queue_name    || nil
-Que.worker_count  = (options.worker_count  || ENV['QUE_WORKER_COUNT']  || Que.worker_count  || 4).to_i
-Que.wake_interval = (options.wake_interval || ENV['QUE_WAKE_INTERVAL'] || Que.wake_interval || 0.1).to_f
-Que.mode          = :async
-
-stop = false
-%w(INT TERM).each { |signal| trap(signal) { stop = true } }
-
-loop do
-  sleep 0.01
-  break if stop
-end
-
-$stdout.puts
-$stdout.puts "Finishing Que's current jobs before exiting..."
-Que.worker_count = 0
-Que.mode = :off
-$stdout.puts "Que's jobs finished, exiting..."
+exit(exit_code)

data/docs/README.md

@@ -1,6 +1,8 @@
 Docs Index
 ===============
 
+TODO: Fix doc links.
+
 - [Advanced Setup](advanced_setup.md#advanced-setup)
   - [Using ActiveRecord Without Rails](advanced_setup.md#using-activerecord-without-rails)
   - [Forking Servers](advanced_setup.md#forking-servers)

data/docs/active_job.md

@@ -0,0 +1,6 @@
+## Using Que With ActiveJob
+
+You can include `Que::ActiveJob::JobExtensions` into your `ApplicationJob` subclass to get support for all of Que's 
+[helper methods](/job_helper_methods.md). These methods will become no-ops if you use a queue adapter that isn't Que, so if you like to use a different adapter in development they shouldn't interfere.
+
+Additionally, including `Que::ActiveJob::JobExtensions` lets you define a run() method that supports keyword arguments.

data/docs/advanced_setup.md

@@ -2,7 +2,7 @@
 
 ### Using ActiveRecord Without Rails
 
-If you're using both Rails and ActiveRecord, the README describes how to get started with Que (which is pretty straightforward, since Que includes a Railtie that handles a lot of setup for you). Otherwise, you'll need to do some manual setup.
+If you're using both Rails and ActiveRecord, the README describes how to get started with Que (which is pretty straightforward, since it includes a Railtie that handles a lot of setup for you). Otherwise, you'll need to do some manual setup.
 
 If you're using ActiveRecord outside of Rails, you'll need to tell Que to piggyback on its connection pool after you've connected to the database:
 
@@ -18,69 +18,22 @@ Then you can queue jobs just as you would in Rails:
 ```ruby
 ActiveRecord::Base.transaction do
   @user = User.create(params[:user])
-  SendRegistrationEmail.enqueue :user_id => @user.id
+  SendRegistrationEmail.enqueue user_id: @user.id
 end
 ```
 
 There are other docs to read if you're using [Sequel](https://github.com/chanks/que/blob/master/docs/using_sequel.md) or [plain Postgres connections](https://github.com/chanks/que/blob/master/docs/using_plain_connections.md) (with no ORM at all) instead of ActiveRecord.
 
-### Forking Servers
-
-If you want to run a worker pool in your web process and you're using a forking webserver like Phusion Passenger (in smart spawning mode), Unicorn or Puma in some configurations, you'll want to set `Que.mode = :off` in your application configuration and only start up the worker pool in the child processes after the DB connection has been reestablished. So, for Puma:
-
-```ruby
-# config/puma.rb
-on_worker_boot do
-  ActiveRecord::Base.establish_connection
-
-  Que.mode = :async
-end
-```
-
-And for Unicorn:
-
-```ruby
-# config/unicorn.rb
-after_fork do |server, worker|
-  ActiveRecord::Base.establish_connection
-
-  Que.mode = :async
-end
-```
-
-And for Phusion Passenger:
-
-```ruby
-# config.ru
-if defined?(PhusionPassenger)
-  PhusionPassenger.on_event(:starting_worker_process) do |forked|
-    if forked
-      Que.mode = :async
-    end
-  end
-end
-```
-
-If there's other setup you want to do for workers, such as setting up the
-configuration, you'll need to do that manually as well.
-
 ### Managing the Jobs Table
 
-After you've connected Que to the database, you can manage the jobs table:
-
-```ruby
-# Create/update the jobs table to the latest schema version:
-Que.migrate!
-```
-
-You'll want to migrate to a specific version if you're using migration files, to ensure that they work the same way even when you upgrade Que in the future:
+After you've connected Que to the database, you can manage the jobs table. You'll want to migrate to a specific version in a migration file, to ensure that they work the same way even when you upgrade Que in the future:
 
 ```ruby
-# Update the schema to version #3.
-Que.migrate! :version => 3
+# Update the schema to version #4.
+Que.migrate! version: 4
 
-# To reverse the migration, drop the jobs table entirely:
-Que.migrate! :version => 0
+# Remove Que's jobs table entirely.
+Que.migrate! version: 0
 ```
 
 There's also a helper method to clear all jobs from the jobs table:
@@ -91,16 +44,6 @@ Que.clear!
 
 ### Other Setup
 
-You'll need to set Que's mode manually:
-
-```ruby
-# Start the worker pool:
-Que.mode = :async
-
-# Or, when testing:
-Que.mode = :sync
-```
-
 Be sure to read the docs on [managing workers](https://github.com/chanks/que/blob/master/docs/managing_workers.md) for more information on using the worker pool.
 
 You'll also want to set up [logging](https://github.com/chanks/que/blob/master/docs/logging.md) and an [error handler](https://github.com/chanks/que/blob/master/docs/error_handling.md) to track errors raised by jobs.

data/docs/command_line_interface.md

@@ -0,0 +1,45 @@
+## Command Line Interface
+
+```
+usage: que [options] [file/to/require] ...
+    -h, --help                       Show this help text.
+    -i, --poll-interval [INTERVAL]   Set maximum interval between polls for available jobs, in seconds (default: 5)
+    -l, --log-level [LEVEL]          Set level at which to log to STDOUT (debug, info, warn, error, fatal) (default: info)
+    -q, --queue-name [NAME]          Set a queue name to work jobs from. Can be passed multiple times. (default: the default queue only)
+    -v, --version                    Print Que version and exit.
+    -w, --worker-count [COUNT]       Set number of workers in process (default: 6)
+        --connection-url [URL]       Set a custom database url to connect to for locking purposes.
+        --log-internals              Log verbosely about Que's internal state. Only recommended for debugging issues
+        --maximum-buffer-size [SIZE] Set maximum number of jobs to be cached in this process awaiting a worker (default: 8)
+        --minimum-buffer-size [SIZE] Set minimum number of jobs to be cached in this process awaiting a worker (default: 2)
+        --wait-period [PERIOD]       Set maximum interval between checks of the in-memory job queue, in milliseconds (default: 50)
+        --worker-priorities [LIST]   List of priorities to assign to workers, unspecified workers take jobs of any priority (default: 10,30,50)
+```
+
+Some explanation of the more unusual options:
+
+### worker-count and worker-priorities
+
+These options dictate the size and priority distribution of the worker pool. The default worker-count is 6 and the default worker-priorities is 10,30,50. This means that the default worker pool will have one worker that only works jobs with priorities under 10, one for priorities under 30, and one for priorities under 50. The leftover workers will work any job.
+
+For example, with these defaults, you could have a large backlog of jobs of priority 100. When a more important job (priority 40) comes in, there's guaranteed to be a free worker. If the process then becomes saturated with jobs of priority 40, and then a priority 20 job comes in, there's guaranteed to be a free worker for it, and so on.
+
+### poll-interval
+
+This option sets the number of seconds the process will wait between polls of the job queue. Jobs that are ready to be worked immediately will be broadcast via the LISTEN/NOTIFY system, so polling is unnecessary for them - polling is only necessary for jobs that are scheduled in the future or which are being delayed due to errors. The default is 5 seconds.
+
+### minimum-buffer-size and maximum-buffer-size
+
+These options set the size of the internal buffer that Que uses to cache job information until it's ready for workers. The default minimum is 2 and the maximum is 8, meaning that the process won't buffer more than 8 jobs that aren't yet ready to be worked, and will only resort to polling if the buffer dips below 2. If you don't want jobs to be buffered at all, you can set both of these values to zero.
+
+### connection-url
+
+This option sets the URL to be used to open a connection to the database for locking purposes. By default, Que will simply use a connection from the connection pool for locking - this option is only useful if your application connections can't use advisory locks - for example, if they're passed through an external connection pool like PgBouncer. In that case, you'll need to use this option to specify your actual database URL so that Que can establish a direct connection.
+
+### wait-period
+
+This option specifies (in milliseconds) how often the locking thread wakes up to check whether the workers have finished jobs, whether it's time to poll, etc. You shouldn't generally need to tweak this, but it may come in handy for some workloads. The default is 50 milliseconds.
+
+### log-internals
+
+This option instructs Que to output a lot of information about its internal state to the logger. It should only be used if it becomes necessary to debug issues.

data/docs/error_handling.md

@@ -7,41 +7,88 @@ If a given job fails repeatedly, Que will retry it at exponentially-increasing i
 ```ruby
 class MyJob < Que::Job
   # Just retry a failed job every 5 seconds:
-  @retry_interval = 5
+  self.retry_interval = 5
 
   # Always retry this job immediately (not recommended, or transient
   # errors will spam your error reporting):
-  @retry_interval = 0
+  self.retry_interval = 0
 
   # Increase the delay by 30 seconds every time this job fails:
-  @retry_interval = proc { |count| count * 30 }
+  self.retry_interval = proc { |count| count * 30 }
 end
 ```
 
-Unlike DelayedJob, however, there is currently no maximum number of failures after which jobs will be deleted. Que's assumption is that if a job is erroring perpetually (and not just transiently), you will want to take action to get the job working properly rather than simply losing it silently.
+There is a maximum_retry_count option for jobs. It defaults to 15 retries, which with the default retry interval means that a job will stop retrying after a little more than two days.
+
+## Error Notifications
 
 If you're using an error notification system (highly recommended, of course), you can hook Que into it by setting a callable as the error notifier:
 
 ```ruby
 Que.error_notifier = proc do |error, job|
-  # Do whatever you want with the error object or job row here.
-
-  # Note that the job passed is not the actual job object, but the hash
-  # representing the job row in the database, which looks like:
+  # Do whatever you want with the error object or job row here. Note that the
+  # job passed is not the actual job object, but the hash representing the job
+  # row in the database, which looks like:
 
   # {
-  #   "queue" => "my_queue",
-  #   "priority" => 100,
-  #   "run_at" => 2015-03-06 11:07:08 -0500,
-  #   "job_id" => 65,
-  #   "job_class" => "MyJob",
-  #   "args" => ['argument', 78],
-  #   "error_count" => 0
+  #   :priority => 100,
+  #   :run_at => "2017-09-15T20:18:52.018101Z",
+  #   :id => 172340879,
+  #   :job_class => "TestJob",
+  #   :error_count => 0,
+  #   :last_error_message => nil,
+  #   :queue => "default",
+  #   :last_error_backtrace => nil,
+  #   :finished_at => nil,
+  #   :expired_at => nil,
+  #   :args => [],
+  #   :data => {}
   # }
 
   # This is done because the job may not have been able to be deserialized
-  # properly, if the name of the job class was changed or the job is being
-  # retrieved and worked by the wrong app. The job argument may also be
-  # nil, if there was a connection failure or something similar.
+  # properly, if the name of the job class was changed or the job class isn't
+  # loaded for some reason. The job argument may also be nil, if there was a
+  # connection failure or something similar.
 end
 ```
+
+## Error-Specific Handling
+
+You can also define a handle_error method in your job, like so:
+
+```ruby
+class MyJob < Que::Job
+  def run(*args)
+    # Your code goes here.
+  end
+
+  def handle_error(error)
+    case error
+    when TemporaryError then retry_in 10.seconds
+    when PermanentError then expire
+    else super # Default (exponential backoff) behavior.
+    end
+  end
+end
+```
+
+The return value of handle_error determines whether the error object is passed to the error notifier. The helper methods like expire and retry_in return true, so these errors will be notified. You can explicitly return false to skip notification.
+
+```ruby
+class MyJob < Que::Job
+  def handle_error(error)
+    case error
+    when AnnoyingError
+      retry_in 10.seconds
+      false
+    when TransientError
+      super
+      error_count > 3
+    else
+      super # Default (exponential backoff) behavior.
+    end
+  end
+end
+```
+
+In this example, AnnoyingError will never be notified, while TransientError will only be notified once it has affected a given job at least three times.

data/docs/inspecting_the_queue.md

@@ -9,106 +9,56 @@ You can call `Que.job_stats` to return some aggregate data on the types of jobs
 ```ruby
 [
   {
-    "job_class"=>"ChargeCreditCard",
-    "count"=>"10",
-    "count_working"=>"4",
-    "count_errored"=>"2",
-    "highest_error_count"=>"5",
-    "oldest_run_at"=>"2014-01-04 21:24:55.817129+00"
+    :job_class=>"ChargeCreditCard",
+    :count=>10,
+    :count_working=>4,
+    :count_errored=>2,
+    :highest_error_count=>5,
+    :oldest_run_at=>2017-09-08 16:13:18 -0400
   },
   {
-    "job_class"=>"SendRegistrationEmail",
-    "count"=>"8",
-    "count_working"=>"0",
-    "count_errored"=>"0",
-    "highest_error_count"=>"0",
-    "oldest_run_at"=>"2014-01-04 22:24:55.81532+00"
+    :job_class=>"SendRegistrationEmail",
+    :count=>1,
+    :count_working=>0,
+    :count_errored=>0,
+    :highest_error_count=>0,
+    :oldest_run_at=>2017-09-08 17:13:18 -0400
   }
 ]
 ```
 
-This tells you that, for instance, there are ten ChargeCreditCard jobs in the queue, four of which are currently being worked, and two of which have experienced errors. One of them has started to process but experienced an error five times. The oldest_run_at is helpful for determining how long jobs have been sitting around, if you have backlog.
-
-### Worker States
-
-You can call `Que.worker_states` to return some information on every worker touching the queue (not just those in the current process). Example output:
-
-```ruby
-[
-  {
-    "priority"=>"2",
-    "run_at"=>"2014-01-04 22:35:55.772324+00",
-    "job_id"=>"4592",
-    "job_class"=>"ChargeCreditCard",
-    "args"=>"[345,56]",
-    "error_count"=>"0",
-    "last_error"=>nil,
-    "pg_backend_pid"=>"1175",
-    "pg_state"=>"idle",
-    "pg_state_changed_at"=>"2014-01-04 22:35:55.777785+00",
-    "pg_last_query"=>"SELECT * FROM users",
-    "pg_last_query_started_at"=>"2014-01-04 22:35:55.777519+00",
-    "pg_transaction_started_at"=>nil,
-    "pg_waiting_on_lock"=>"f"
-  }
-]
-```
-
-In this case, there is only one worker currently working the queue. The first seven fields are the attributes of the job it is currently running. The next seven fields are information about that worker's Postgres connection, and are taken from `pg_stat_activity` - see [Postgres' documentation](http://www.postgresql.org/docs/current/static/monitoring-stats.html#PG-STAT-ACTIVITY-VIEW) for more information on interpreting these fields.
-
-* `pg_backend_pid` - The pid of the Postgres process serving this worker. This is useful if you wanted to kill that worker's connection, for example, by running "SELECT pg_terminate_backend(1175)". This would free up the job to be attempted by another worker.
-* `pg_state` - The state of the Postgres backend. It may be "active" if the worker is currently running a query or "idle"/"idle in transaction" if it is not. It may also be in one of a few other less common states.
-* `pg_state_changed_at` - The timestamp for when the backend's state was last changed. If the backend is idle, this would reflect the time that the last query finished.
-* `pg_last_query` - The text of the current or most recent query that the worker sent to the database.
-* `pg_last_query_started_at` - The timestamp for when the last query began to run.
-* `pg_transaction_started_at` - The timestamp for when the worker's current transaction (if any) began.
-* `pg_waiting_on_lock` - Whether or not the worker is waiting for a lock in Postgres to be released.
+This tells you that, for instance, there are ten ChargeCreditCard jobs in the queue, four of which are currently being worked, and two of which have experienced errors. One of them has started to process but experienced an error five times. The oldest_run_at is helpful for determining how long jobs have been sitting around, if you have a large backlog.
 
 ### Custom Queries
 
-If you want to query the jobs table yourself to see what's been queued or to check the state of various jobs, you can always use Que to execute whatever SQL you want:
+If you're using ActiveRecord or Sequel, Que ships with models that wrap the job queue so you can write your own logic to inspect it. They include some helpful scopes to write your queries - see the gem source for a complete accounting.
 
-```ruby
-Que.execute("select count(*) from que_jobs") #=> [{"count"=>"492"}]
-```
+#### ActiveRecord Example
 
-If you want to use ActiveRecord's features when querying, you can define your own model around Que's job table:
+``` ruby
+# app/models/que_job.rb
 
-```ruby
-class QueJob < ActiveRecord::Base
+require 'que/active_record/model'
+
+class QueJob < Que::ActiveRecord::Model
 end
 
-# Or:
+QueJob.finished.to_sql # => "SELECT \"que_jobs\".* FROM \"que_jobs\" WHERE (\"que_jobs\".\"finished_at\" IS NOT NULL)"
 
-class MyJob < ActiveRecord::Base
-  self.table_name = :que_jobs
-end
+# You could also name the model whatever you like, or just query from
+# Que::ActiveRecord::Model directly if you don't need to write your own model
+# logic.
 ```
 
-Then you can query just as you would with any other model. Since the jobs table has a composite primary key, however, you probably won't be able to update or destroy jobs this way, though.
+#### Sequel Example
 
-If you're using Sequel, you can use the same technique:
+``` ruby
+# app/models/que_job.rb
 
-```ruby
-class QueJob < Sequel::Model
-end
-
-# Or:
+require 'que/sequel/model'
 
-class MyJob < Sequel::Model(:que_jobs)
+class QueJob < Que::Sequel::Model
 end
-```
 
-And note that Sequel *does* support composite primary keys:
-
-```ruby
-job = QueJob.where(:job_class => "ChargeCreditCard").first
-job.priority = 1
-job.save
-```
-
-Or, you can just use Sequel's dataset methods:
-
-```ruby
-DB[:que_jobs].where{priority > 3}.all
+QueJob.finished # => #<Sequel::Postgres::Dataset: "SELECT * FROM \"public\".\"que_jobs\" WHERE (\"public\".\"que_jobs\".\"finished_at\" IS NOT NULL)">
 ```