que 0.11.3 → 2.2.0

data/lib/que/sql.rb

@@ -1,152 +0,0 @@
-# frozen_string_literal: true
-
-module Que
-  SQL = {
-    # Locks a job using a Postgres recursive CTE [1].
-    #
-    # As noted by the Postgres documentation, it may be slightly easier to
-    # think about this expression as iteration rather than recursion, despite
-    # the `RECURSION` nomenclature defined by the SQL standards committee.
-    # Recursion is used here so that jobs in the table can be iterated one-by-
-    # one until a lock can be acquired, where a non-recursive `SELECT` would
-    # have the undesirable side-effect of locking multiple jobs at once. i.e.
-    # Consider that the following would have the worker lock *all* unlocked
-    # jobs:
-    #
-    #   SELECT (j).*, pg_try_advisory_lock((j).job_id) AS locked
-    #   FROM que_jobs AS j;
-    #
-    # The CTE will initially produce an "anchor" from the non-recursive term
-    # (i.e. before the `UNION`), and then use it as the contents of the
-    # working table as it continues to iterate through `que_jobs` looking for
-    # a lock. The jobs table has a sort on (priority, run_at, job_id) which
-    # allows it to walk the jobs table in a stable manner. As noted above, the
-    # recursion examines one job at a time so that it only ever acquires a
-    # single lock.
-    #
-    # The recursion has two possible end conditions:
-    #
-    # 1. If a lock *can* be acquired, it bubbles up to the top-level `SELECT`
-    #    outside of the `job` CTE which stops recursion because it is
-    #    constrained with a `LIMIT` of 1.
-    #
-    # 2. If a lock *cannot* be acquired, the recursive term of the expression
-    #    (i.e. what's after the `UNION`) will return an empty result set
-    #    because there are no more candidates left that could possibly be
-    #    locked. This empty result automatically ends recursion.
-    #
-    # Note that this query can be easily modified to lock any number of jobs
-    # by tweaking the LIMIT clause in the main SELECT statement.
-    #
-    # [1] http://www.postgresql.org/docs/devel/static/queries-with.html
-    #
-    # Thanks to RhodiumToad in #postgresql for help with the original version
-    # of the job lock CTE.
-    :lock_job => %{
-      WITH RECURSIVE jobs AS (
-        SELECT (j).*, pg_try_advisory_lock((j).job_id) AS locked
-        FROM (
-          SELECT j
-          FROM que_jobs AS j
-          WHERE queue = $1::text
-          AND run_at <= now()
-          ORDER BY priority, run_at, job_id
-          LIMIT 1
-        ) AS t1
-        UNION ALL (
-          SELECT (j).*, pg_try_advisory_lock((j).job_id) AS locked
-          FROM (
-            SELECT (
-              SELECT j
-              FROM que_jobs AS j
-              WHERE queue = $1::text
-              AND run_at <= now()
-              AND (priority, run_at, job_id) > (jobs.priority, jobs.run_at, jobs.job_id)
-              ORDER BY priority, run_at, job_id
-              LIMIT 1
-            ) AS j
-            FROM jobs
-            WHERE jobs.job_id IS NOT NULL
-            LIMIT 1
-          ) AS t1
-        )
-      )
-      SELECT queue, priority, run_at, job_id, job_class, args, error_count
-      FROM jobs
-      WHERE locked
-      LIMIT 1
-    }.freeze,
-
-    :check_job => %{
-      SELECT 1 AS one
-      FROM   que_jobs
-      WHERE  queue    = $1::text
-      AND    priority = $2::smallint
-      AND    run_at   = $3::timestamptz
-      AND    job_id   = $4::bigint
-    }.freeze,
-
-    :set_error => %{
-      UPDATE que_jobs
-      SET error_count = $1::integer,
-          run_at      = now() + $2::bigint * '1 second'::interval,
-          last_error  = $3::text
-      WHERE queue     = $4::text
-      AND   priority  = $5::smallint
-      AND   run_at    = $6::timestamptz
-      AND   job_id    = $7::bigint
-    }.freeze,
-
-    :insert_job => %{
-      INSERT INTO que_jobs
-      (queue, priority, run_at, job_class, args)
-      VALUES
-      (coalesce($1, '')::text, coalesce($2, 100)::smallint, coalesce($3, now())::timestamptz, $4::text, coalesce($5, '[]')::json)
-      RETURNING *
-    }.freeze,
-
-    :destroy_job => %{
-      DELETE FROM que_jobs
-      WHERE queue    = $1::text
-      AND   priority = $2::smallint
-      AND   run_at   = $3::timestamptz
-      AND   job_id   = $4::bigint
-    }.freeze,
-
-    :job_stats => %{
-      SELECT queue,
-             job_class,
-             count(*)                    AS count,
-             count(locks.job_id)         AS count_working,
-             sum((error_count > 0)::int) AS count_errored,
-             max(error_count)            AS highest_error_count,
-             min(run_at)                 AS oldest_run_at
-      FROM que_jobs
-      LEFT JOIN (
-        SELECT (classid::bigint << 32) + objid::bigint AS job_id
-        FROM pg_locks
-        WHERE locktype = 'advisory'
-      ) locks USING (job_id)
-      GROUP BY queue, job_class
-      ORDER BY count(*) DESC
-    }.freeze,
-
-    :worker_states => %{
-      SELECT que_jobs.*,
-             pg.pid          AS pg_backend_pid,
-             pg.state        AS pg_state,
-             pg.state_change AS pg_state_changed_at,
-             pg.query        AS pg_last_query,
-             pg.query_start  AS pg_last_query_started_at,
-             pg.xact_start   AS pg_transaction_started_at,
-             pg.waiting      AS pg_waiting_on_lock
-      FROM que_jobs
-      JOIN (
-        SELECT (classid::bigint << 32) + objid::bigint AS job_id, pg_stat_activity.*
-        FROM pg_locks
-        JOIN pg_stat_activity USING (pid)
-        WHERE locktype = 'advisory'
-      ) pg USING (job_id)
-    }.freeze
-  }.freeze
-end

data/spec/adapters/active_record_spec.rb

@@ -1,152 +0,0 @@
-# frozen_string_literal: true
-
-# Don't run these specs in JRuby until jruby-pg is compatible with ActiveRecord.
-unless defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
-
-  require 'spec_helper'
-  require 'active_record'
-
-  if ActiveRecord.version.release >= Gem::Version.new('4.2')
-    ActiveRecord::Base.raise_in_transactional_callbacks = true
-  end
-  ActiveRecord::Base.establish_connection(QUE_URL)
-
-  Que.connection = ActiveRecord
-  QUE_ADAPTERS[:active_record] = Que.adapter
-
-  describe "Que using the ActiveRecord adapter" do
-    before { Que.adapter = QUE_ADAPTERS[:active_record] }
-
-    it_behaves_like "a multi-threaded Que adapter"
-
-    it "should use the same connection that ActiveRecord does" do
-      begin
-        class ActiveRecordJob < Que::Job
-          def run
-            $pid1 = Integer(Que.execute("select pg_backend_pid()").first['pg_backend_pid'])
-            $pid2 = Integer(ActiveRecord::Base.connection.select_value("select pg_backend_pid()"))
-          end
-        end
-
-        ActiveRecordJob.enqueue
-        Que::Job.work
-
-        $pid1.should == $pid2
-      ensure
-        $pid1 = $pid2 = nil
-      end
-    end
-
-    context "if the connection goes down and is reconnected" do
-      before do
-        Que::Job.enqueue
-        ActiveRecord::Base.connection.reconnect!
-      end
-
-      it "should recreate the prepared statements" do
-        expect { Que::Job.enqueue }.not_to raise_error
-
-        DB[:que_jobs].count.should == 2
-      end
-
-      it "should work properly even in a transaction" do
-        ActiveRecord::Base.transaction do
-          expect { Que::Job.enqueue }.not_to raise_error
-        end
-
-        DB[:que_jobs].count.should == 2
-      end
-
-      it "should log this extraordinary event" do
-        $logger.messages.clear
-        Que::Job.enqueue
-        $logger.messages.count.should == 1
-        message = JSON.load($logger.messages.first)
-        message['lib'].should == 'que'
-        message['event'].should == 'reprepare_statement'
-        message['name'].should == 'insert_job'
-      end
-    end
-
-    it "should instantiate args as ActiveSupport::HashWithIndifferentAccess" do
-      begin
-        # Mimic the setting in the Railtie.
-        Que.json_converter = :with_indifferent_access.to_proc
-
-        ArgsJob.enqueue :param => 2
-        Que::Job.work
-        $passed_args.first[:param].should == 2
-        $passed_args.first.should be_an_instance_of ActiveSupport::HashWithIndifferentAccess
-      ensure
-        Que.json_converter = Que::INDIFFERENTIATOR
-      end
-    end
-
-    it "should support Rails' special extensions for times" do
-      Que.mode = :async
-      Que.worker_count = 4
-      sleep_until { Que::Worker.workers.all? &:sleeping? }
-
-      Que::Job.enqueue :run_at => 1.minute.ago
-      DB[:que_jobs].get(:run_at).should be_within(3).of Time.now - 60
-
-      Que.wake_interval = 0.005.seconds
-      sleep_until { DB[:que_jobs].empty? }
-    end
-
-    it "should wake up a Worker after queueing a job in async mode, waiting for a transaction to commit if necessary" do
-      Que.mode = :async
-      Que.worker_count = 4
-      sleep_until { Que::Worker.workers.all? &:sleeping? }
-
-      # Wakes a worker immediately when not in a transaction.
-      Que::Job.enqueue
-      sleep_until { Que::Worker.workers.all?(&:sleeping?) && DB[:que_jobs].empty? }
-
-      ActiveRecord::Base.transaction do
-        Que::Job.enqueue
-        Que::Worker.workers.each { |worker| worker.should be_sleeping }
-      end
-      sleep_until { Que::Worker.workers.all?(&:sleeping?) && DB[:que_jobs].empty? }
-
-      # Do nothing when queueing with a specific :run_at.
-      BlockJob.enqueue :run_at => Time.now
-      Que::Worker.workers.each { |worker| worker.should be_sleeping }
-    end
-
-    it "should be able to survive an ActiveRecord::Rollback without raising an error" do
-      ActiveRecord::Base.transaction do
-        Que::Job.enqueue
-        raise ActiveRecord::Rollback, "Call tech support!"
-      end
-      DB[:que_jobs].count.should be 0
-    end
-
-    it "should be able to tell when it's in an ActiveRecord transaction" do
-      Que.adapter.should_not be_in_transaction
-      ActiveRecord::Base.transaction do
-        Que.adapter.should be_in_transaction
-      end
-    end
-
-    it "should not leak connections to other databases when using ActiveRecord's multiple database support" do
-      class SecondDatabaseModel < ActiveRecord::Base
-        establish_connection(QUE_URL)
-      end
-
-      SecondDatabaseModel.clear_active_connections!
-      SecondDatabaseModel.connection_handler.active_connections?.should == false
-
-      class SecondDatabaseModelJob < Que::Job
-        def run(*args)
-          SecondDatabaseModel.connection.execute("SELECT 1")
-        end
-      end
-
-      SecondDatabaseModelJob.enqueue
-      Que::Job.work
-
-      SecondDatabaseModel.connection_handler.active_connections?.should == false
-    end
-  end
-end

data/spec/adapters/connection_pool_spec.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-require 'connection_pool'
-
-Que.connection = QUE_SPEC_CONNECTION_POOL = ConnectionPool.new &NEW_PG_CONNECTION
-QUE_ADAPTERS[:connection_pool] = Que.adapter
-
-describe "Que using the ConnectionPool adapter" do
-  before { Que.adapter = QUE_ADAPTERS[:connection_pool] }
-
-  it_behaves_like "a multi-threaded Que adapter"
-
-  it "should be able to tell when it's already in a transaction" do
-    Que.adapter.should_not be_in_transaction
-    QUE_SPEC_CONNECTION_POOL.with do |conn|
-      conn.async_exec "BEGIN"
-      Que.adapter.should be_in_transaction
-      conn.async_exec "COMMIT"
-    end
-  end
-end

data/spec/adapters/pg_spec.rb

@@ -1,41 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-describe "Que using a bare PG connection" do
-  it_behaves_like "a Que adapter"
-
-  it "should synchronize access to that connection" do
-    lock   = Que.adapter.lock
-    q1, q2 = Queue.new, Queue.new
-
-    thread1 = Thread.new do
-      Que.adapter.checkout do
-        q1.push nil
-        q2.pop
-      end
-    end
-
-    q1.pop
-
-    thread2 = Thread.new do
-      Que.adapter.checkout do
-        q1.push nil
-        q2.pop
-      end
-    end
-
-    sleep_until { thread2.status == 'sleep' }
-
-    thread1.should be_alive
-    thread2.should be_alive
-
-    lock.send(:instance_variable_get, :@mon_owner).should == thread1
-    q2.push nil
-    q1.pop
-    lock.send(:instance_variable_get, :@mon_owner).should == thread2
-    q2.push nil
-    thread1.join
-    thread2.join
-  end
-end

data/spec/adapters/pond_spec.rb

@@ -1,22 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-require 'pond'
-
-Que.connection = QUE_SPEC_POND = Pond.new &NEW_PG_CONNECTION
-QUE_ADAPTERS[:pond] = Que.adapter
-
-describe "Que using the Pond adapter" do
-  before { Que.adapter = QUE_ADAPTERS[:pond] }
-
-  it_behaves_like "a multi-threaded Que adapter"
-
-  it "should be able to tell when it's already in a transaction" do
-    Que.adapter.should_not be_in_transaction
-    QUE_SPEC_POND.checkout do |conn|
-      conn.async_exec "BEGIN"
-      Que.adapter.should be_in_transaction
-      conn.async_exec "COMMIT"
-    end
-  end
-end

data/spec/adapters/sequel_spec.rb

@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-Que.connection = SEQUEL_ADAPTER_DB = Sequel.connect(QUE_URL)
-QUE_ADAPTERS[:sequel] = Que.adapter
-
-describe "Que using the Sequel adapter" do
-  before { Que.adapter = QUE_ADAPTERS[:sequel] }
-
-  it_behaves_like "a multi-threaded Que adapter"
-
-  it "should use the same connection that Sequel does" do
-    begin
-      class SequelJob < Que::Job
-        def run
-          $pid1 = Integer(Que.execute("select pg_backend_pid()").first['pg_backend_pid'])
-          $pid2 = Integer(SEQUEL_ADAPTER_DB['select pg_backend_pid()'].get)
-        end
-      end
-
-      SequelJob.enqueue
-      Que::Job.work
-
-      $pid1.should == $pid2
-    ensure
-      $pid1 = $pid2 = nil
-    end
-  end
-
-  it "should wake up a Worker after queueing a job in async mode, waiting for a transaction to commit if necessary" do
-    Que.mode = :async
-    Que.worker_count = 4
-    sleep_until { Que::Worker.workers.all? &:sleeping? }
-
-    # Wakes a worker immediately when not in a transaction.
-    Que::Job.enqueue
-    sleep_until { Que::Worker.workers.all?(&:sleeping?) && DB[:que_jobs].empty? }
-
-    SEQUEL_ADAPTER_DB.transaction do
-      Que::Job.enqueue
-      Que::Worker.workers.each { |worker| worker.should be_sleeping }
-    end
-    sleep_until { Que::Worker.workers.all?(&:sleeping?) && DB[:que_jobs].empty? }
-
-    # Do nothing when queueing with a specific :run_at.
-    BlockJob.enqueue :run_at => Time.now
-    Que::Worker.workers.each { |worker| worker.should be_sleeping }
-  end
-
-  it "should be able to tell when it's in a Sequel transaction" do
-    Que.adapter.should_not be_in_transaction
-    SEQUEL_ADAPTER_DB.transaction do
-      Que.adapter.should be_in_transaction
-    end
-  end
-end

data/spec/gemfiles/Gemfile1

@@ -1,18 +0,0 @@
-source 'https://rubygems.org'
-
-gem 'que', path: '../..'
-
-group :development, :test do
-  gem 'rake'
-
-  gem 'activerecord',    '~> 3.2', :require => nil
-  gem 'sequel',          '~> 3', :require => nil
-  gem 'connection_pool', :require => nil
-  gem 'pg',              :require => nil, :platform => :ruby
-  gem 'pg_jruby',        :require => nil, :platform => :jruby
-end
-
-group :test do
-  gem 'rspec', '~> 2.14.1'
-  gem 'pry'
-end

data/spec/gemfiles/Gemfile2

@@ -1,18 +0,0 @@
-source 'https://rubygems.org'
-
-gem 'que', path: "../.."
-
-group :development, :test do
-  gem 'rake'
-
-  gem 'activerecord',    '~> 4.0', :require => nil
-  gem 'sequel',          '~> 4', :require => nil
-  gem 'connection_pool', :require => nil
-  gem 'pg',              :require => nil, :platform => :ruby
-  gem 'pg_jruby',        :require => nil, :platform => :jruby
-end
-
-group :test do
-  gem 'rspec', '~> 2.14.1'
-  gem 'pry'
-end

data/spec/spec_helper.rb

@@ -1,118 +0,0 @@
-# frozen_string_literal: true
-
-require 'que'
-require 'uri'
-require 'pg'
-require 'logger'
-require 'json'
-
-Dir['./spec/support/**/*.rb'].sort.each &method(:require)
-
-
-
-# Handy constants for initializing PG connections:
-QUE_URL = ENV['DATABASE_URL'] || 'postgres://postgres:@localhost/que-test'
-
-NEW_PG_CONNECTION = proc do
-  uri = URI.parse(QUE_URL)
-  pg = PG::Connection.open :host     => uri.host,
-                           :user     => uri.user,
-                           :password => uri.password,
-                           :port     => uri.port || 5432,
-                           :dbname   => uri.path[1..-1]
-
-  # Avoid annoying NOTICE messages in specs.
-  pg.async_exec "SET client_min_messages TO 'warning'"
-  pg
-end
-
-
-
-# Adapters track which statements have been prepared for their connections,
-# and if Que.connection= is called before each spec, we're constantly creating
-# new adapters and losing that information, which is bad. So instead, we hang
-# onto a few adapters and assign them using Que.adapter= as needed. The plain
-# pg adapter is the default.
-
-# Also, let Que initialize the adapter itself, to make sure that the
-# recognition logic works. Similar code can be found in the adapter specs.
-Que.connection = NEW_PG_CONNECTION.call
-QUE_ADAPTERS = {:pg => Que.adapter}
-
-
-
-# We use Sequel to examine the database in specs.
-require 'sequel'
-DB = Sequel.connect(QUE_URL)
-
-
-
-# Reset the table to the most up-to-date version.
-DB.drop_table? :que_jobs
-Que::Migrations.migrate!
-
-
-
-# Set up a dummy logger.
-Que.logger = $logger = Object.new
-$logger_mutex = Mutex.new # Protect against rare errors on Rubinius/JRuby.
-
-def $logger.messages
-  @messages ||= []
-end
-
-def $logger.method_missing(m, message)
-  $logger_mutex.synchronize { messages << message }
-end
-
-# Object includes Kernel#warn which is not what we expect, so remove:
-def $logger.warn(message)
-  method_missing(:warn, message)
-end
-
-
-
-# Helper to display spec descriptions.
-description_builder = -> hash do
-  if g = hash[:example_group]
-    "#{description_builder.call(g)} #{hash[:description_args].first}"
-  else
-    hash[:description_args].first
-  end
-end
-
-stdout = Logger.new(STDOUT)
-
-RSpec.configure do |config|
-  config.around do |spec|
-    # Figure out which spec is about to run, for logging purposes.
-    data = example.metadata
-    desc = description_builder.call(data)
-    line = "rspec #{data[:file_path]}:#{data[:line_number]}"
-
-    # Optionally log to STDOUT which spec is about to run. This is noisy, but
-    # helpful in identifying hanging specs.
-    stdout.info "Running spec: #{desc} @ #{line}" if ENV['LOG_SPEC']
-
-    Que.adapter = QUE_ADAPTERS[:pg]
-
-    Que.worker_count = 0
-    Que.mode = :async
-    Que.wake_interval = nil
-
-    $logger.messages.clear
-
-    spec.run
-
-    Que.worker_count = 0
-    Que.mode = :off
-    Que.wake_interval = nil
-
-    DB[:que_jobs].delete
-
-    # A bit of lint: make sure that no advisory locks are left open.
-    unless DB[:pg_locks].where(:locktype => 'advisory').empty?
-      stdout.info "Advisory lock left open: #{desc} @ #{line}"
-    end
-  end
-end

data/spec/support/helpers.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-# Helper for testing threaded code.
-QUE_TEST_TIMEOUT ||= 2
-def sleep_until(timeout = QUE_TEST_TIMEOUT)
-  deadline = Time.now + timeout
-  loop do
-    break if yield
-    raise "Thing never happened!" if Time.now > deadline
-    sleep 0.01
-  end
-end
-
-def suppress_warnings
-  original_verbosity, $VERBOSE = $VERBOSE, nil
-  yield
-ensure
-  $VERBOSE = original_verbosity
-end

data/spec/support/jobs.rb

@@ -1,35 +0,0 @@
-# frozen_string_literal: true
-
-# Common Job classes for use in specs.
-
-# Handy for blocking in the middle of processing a job.
-class BlockJob < Que::Job
-  def run
-    $q1.push nil
-    $q2.pop
-  end
-end
-
-RSpec.configure do |config|
-  config.before { $q1, $q2 = Queue.new, Queue.new }
-end
-
-
-
-class ErrorJob < Que::Job
-  def run
-    raise "ErrorJob!"
-  end
-end
-
-
-
-class ArgsJob < Que::Job
-  def run(*args)
-    $passed_args = args
-  end
-end
-
-RSpec.configure do |config|
-  config.before { $passed_args = nil }
-end

data/spec/support/shared_examples/adapter.rb

@@ -1,37 +0,0 @@
-# frozen_string_literal: true
-
-shared_examples "a Que adapter" do
-  it "should be able to execute arbitrary SQL and return indifferent hashes" do
-    result = Que.execute("SELECT 1 AS one")
-    result.should == [{'one'=>1}]
-    result.first[:one].should == 1
-  end
-
-  it "should be able to execute multiple SQL statements in one string" do
-    Que.execute("SELECT 1 AS one; SELECT 1 AS one")
-  end
-
-  it "should be able to queue and work a job" do
-    Que::Job.enqueue
-    result = Que::Job.work
-    result[:event].should == :job_worked
-    result[:job][:job_class].should == 'Que::Job'
-  end
-
-  it "should yield the same Postgres connection for the duration of the block" do
-    Que.adapter.checkout do |conn|
-      conn.should be_a PG::Connection
-      pid1 = Que.execute "SELECT pg_backend_pid()"
-      pid2 = Que.execute "SELECT pg_backend_pid()"
-      pid1.should == pid2
-    end
-  end
-
-  it "should allow nested checkouts" do
-    Que.adapter.checkout do |a|
-      Que.adapter.checkout do |b|
-        a.object_id.should == b.object_id
-      end
-    end
-  end
-end