queue_classic 2.3.0beta → 3.0.0beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c38026847651ea48626d07f07fffd717a56bd13f
-  data.tar.gz: 49543473a45f7b5ca199f5675fa82c8d97dafbfd
+  metadata.gz: 9ae7f3fa42909c77c861db2ef125fab2ba01205b
+  data.tar.gz: a41224c39e1966276a5a66403b705426c5d3ff63
 SHA512:
-  metadata.gz: 67bac654f7508dd9104cacd47fc9721c5c4e28550b68c08e9bbd3d5ae870b80415172141a9766b5e341e9fb3b2cc72cd7a28bdc0719a8fcbbc7bba25353e5780
-  data.tar.gz: 95335e3b6990954cb10f99d157fe7a8b175da9e27d09b25c57a1cd84eee15f62dc9ed607bba286eb3fb75a4785add817c2f3c9c5408895606a8f34dc6a35b022
+  metadata.gz: 4a0d7b6b9f80e240ce3ac76c3fd488870ca648439c66917eceec454c7595dbbac7193d17a91ea099b763f25e39f76038761521d799ae3eaf4b5f09ed5c64b995
+  data.tar.gz: 92ff9563d1b98f03b096704524059a8d3f72d4ccd2626517af62b5067784641ff4eccb4f8864a9830fdc95ec06a3d8b0e5d6cc04c47656a247c2c0b40ad1d6df

data/lib/generators/queue_classic/install_generator.rb

@@ -8,8 +8,7 @@ module QC
 
     namespace "queue_classic:install"
     self.source_paths << File.join(File.dirname(__FILE__), 'templates')
-    desc 'Generates (but does not run) a migration to add ' +
-      'a queue_classic table.'
+    desc 'Generates (but does not run) a migration to add a queue_classic table.'
 
     def self.next_migration_number(dirname)
       next_migration_number = current_migration_number(dirname) + 1
@@ -17,8 +16,7 @@ module QC
     end
 
     def create_migration_file
-      migration_template 'add_queue_classic.rb',
-        'db/migrate/add_queue_classic.rb'
+      migration_template 'add_queue_classic.rb', 'db/migrate/add_queue_classic.rb'
     end
   end
 end

data/lib/queue_classic/conn_adapter.rb

@@ -0,0 +1,106 @@
+require 'thread'
+require 'uri'
+require 'pg'
+
+module QC
+  class ConnAdapter
+
+    attr_accessor :connection
+    def initialize(c=nil)
+      @connection = c.nil? ? establish_new : validate!(c)
+      @mutex = Mutex.new
+    end
+
+    def execute(stmt, *params)
+      @mutex.synchronize do
+        QC.log(:at => "exec_sql", :sql => stmt.inspect)
+        begin
+          params = nil if params.empty?
+          r = @connection.exec(stmt, params)
+          result = []
+          r.each {|t| result << t}
+          result.length > 1 ? result : result.pop
+        rescue PGError => e
+          QC.log(:error => e.inspect)
+          @connection.reset
+          raise
+        end
+      end
+    end
+
+    def wait(time, *channels)
+      @mutex.synchronize do
+        listen_cmds = channels.map {|c| 'LISTEN "' + c + '"'}
+        @connection.exec(listen_cmds.join(';'))
+        wait_for_notify(time)
+        unlisten_cmds = channels.map {|c| 'UNLISTEN "' + c +'"'}
+        @connection.exec(unlisten_cmds.join(';'))
+        drain_notify
+      end
+    end
+
+    def disconnect
+      @mutex.synchronize do
+        begin
+          @connection.close
+        rescue => e
+          QC.log(:at => 'disconnect', :error => e.message)
+        end
+      end
+    end
+
+    private
+
+    def wait_for_notify(t)
+      Array.new.tap do |msgs|
+        @connection.wait_for_notify(t) {|event, pid, msg| msgs << msg}
+      end
+    end
+
+    def drain_notify
+      until @connection.notifies.nil?
+        QC.log(:at => "drain_notifications")
+      end
+    end
+
+    def validate!(c)
+      return c if c.is_a?(PG::Connection)
+      klass = c.class
+      err = "connection must be an instance of PG::Connection, but was #{c}"
+      raise(ArgumentError, err)
+    end
+
+    def establish_new
+      QC.log(:at => "establish_conn")
+      conn = PGconn.connect(*normalize_db_url(db_url))
+      if conn.status != PGconn::CONNECTION_OK
+        QC.log(:error => conn.error)
+      end
+      conn.exec("SET application_name = '#{QC::APP_NAME}'")
+      conn
+    end
+
+    def normalize_db_url(url)
+      host = url.host
+      host = host.gsub(/%2F/i, '/') if host
+
+      [
+       host, # host or percent-encoded socket path
+       url.port || 5432,
+       nil, '', #opts, tty
+       url.path.gsub("/",""), # database name
+       url.user,
+       url.password
+      ]
+    end
+
+    def db_url
+      return @db_url if @db_url
+      url = ENV["QC_DATABASE_URL"] ||
+            ENV["DATABASE_URL"]    ||
+            raise(ArgumentError, "missing QC_DATABASE_URL or DATABASE_URL")
+      @db_url = URI.parse(url)
+    end
+
+  end
+end

data/lib/queue_classic/queue.rb

@@ -1,35 +1,35 @@
 require 'queue_classic'
-require 'queue_classic/conn'
+require 'queue_classic/conn_adapter'
 require 'json'
 
 module QC
   class Queue
-    TABLE_NAME = "queue_classic_jobs"
-    # Each row in the table will have a column that
-    # notes the queue.
-    QUEUE_NAME = ENV["QUEUE"] || "default"
-    # Set this to 1 for strict FIFO.
-    TOP_BOUND = (ENV["QC_TOP_BOUND"] || 9).to_i
 
+    attr_reader :name, :top_bound
+    def initialize(name, top_bound=nil)
+      @name = name
+      @top_bound = top_bound || QC::TOP_BOUND
+    end
+
+    def conn_adapter=(a)
+      @adapter = a
+    end
 
-    attr_reader :conn, :name, :top_bound
-    def initialize(opts={})
-      @conn       = opts[:conn]       || Conn.new
-      @name       = opts[:name]       || QUEUE_NAME
-      @top_bound  = opts[:top_bound]  || TOP_BOUND
+    def conn_adapter
+      @adapter ||= QC.default_conn_adapter
     end
 
     def enqueue(method, *args)
       QC.log_yield(:measure => 'queue.enqueue') do
         s="INSERT INTO #{TABLE_NAME} (q_name, method, args) VALUES ($1, $2, $3)"
-        res = conn.execute(s, name, method, JSON.dump(args))
+        res = conn_adapter.execute(s, name, method, JSON.dump(args))
       end
     end
 
     def lock
       QC.log_yield(:measure => 'queue.lock') do
         s = "SELECT * FROM lock_head($1, $2)"
-        if r = conn.execute(s, name, top_bound)
+        if r = conn_adapter.execute(s, name, top_bound)
           {:id => r["id"],
             :method => r["method"],
             :args => JSON.parse(r["args"])}
@@ -37,30 +37,23 @@ module QC
       end
     end
 
-    def wait
-      QC.log_yield(:measure => 'queue.wait') do
-        conn.wait(name)
-      end
-    end
-
     def delete(id)
       QC.log_yield(:measure => 'queue.delete') do
-        s = "DELETE FROM #{TABLE_NAME} where id = $1"
-        conn.execute(s, id)
+        conn_adapter.execute("DELETE FROM #{TABLE_NAME} where id = $1", id)
       end
     end
 
     def delete_all
       QC.log_yield(:measure => 'queue.delete_all') do
         s = "DELETE FROM #{TABLE_NAME} WHERE q_name = $1"
-        conn.execute(s, name)
+        conn_adapter.execute(s, name)
       end
     end
 
     def count
       QC.log_yield(:measure => 'queue.count') do
         s = "SELECT COUNT(*) FROM #{TABLE_NAME} WHERE q_name = $1"
-        r = conn.execute(s, name)
+        r = conn_adapter.execute(s, name)
         r["count"].to_i
       end
     end

data/lib/queue_classic/setup.rb

@@ -1,5 +1,3 @@
-require 'queue_classic/conn'
-
 module QC
   module Setup
     Root = File.expand_path("../..", File.dirname(__FILE__))
@@ -7,16 +5,18 @@ module QC
     CreateTable = File.join(Root, "/sql/create_table.sql")
     DropSqlFunctions = File.join(Root, "/sql/drop_ddl.sql")
 
-    def self.create(conn=nil)
-      conn ||= Conn.new
+    def self.create(c=nil)
+      conn = QC::ConnAdapter.new(c)
       conn.execute(File.read(CreateTable))
       conn.execute(File.read(SqlFunctions))
+      conn.disconnect if c.nil? #Don't close a conn we didn't create.
     end
 
-    def self.drop(conn=nil)
-      conn ||= Conn.new
+    def self.drop(c=nil)
+      conn = QC::ConnAdapter.new(c)
       conn.execute("DROP TABLE IF EXISTS queue_classic_jobs CASCADE")
       conn.execute(File.read(DropSqlFunctions))
+      conn.disconnect if c.nil? #Don't close a conn we didn't create.
     end
   end
 end

data/lib/queue_classic/tasks.rb

@@ -1,42 +1,31 @@
-if Rake::Task.task_defined? "qc:count"
-  ActiveSupport::Deprecation.warn <<-MSG
-queue_classic Rake tasks are now loaded automatically for Rails applications.
-Loading the tasks yourself is deprecated. Please update your Rakefile and remove:
+task :environment
 
-  require 'queue_classic'
-  require 'queue_classic/tasks'
-
-  MSG
-else
-  task :environment
+namespace :jobs do
+  desc "Alias for qc:work"
+  task :work  => "qc:work"
+end
 
-  namespace :jobs do
-    desc "Alias for qc:work"
-    task :work  => "qc:work"
+namespace :qc do
+  desc "Start a new worker for the (default or $QUEUE) queue"
+  task :work  => :environment do
+    trap('INT') {exit}
+    trap('TERM') {@worker.stop}
+    @worker = QC::Worker.new
+    @worker.start
   end
 
-  namespace :qc do
-    desc "Start a new worker for the (default or $QUEUE) queue"
-    task :work  => :environment do
-      trap('INT') {exit}
-      trap('TERM') {@worker.stop}
-      @worker = QC::Worker.new
-      @worker.start
-    end
-
-    desc "Returns the number of jobs in the (default or QUEUE) queue"
-    task :count => :environment do
-      puts QC::Worker.new.queue.count
-    end
+  desc "Returns the number of jobs in the (default or QUEUE) queue"
+  task :count => :environment do
+    puts QC::Worker.new.queue.count
+  end
 
-    desc "Setup queue_classic tables and functions in database"
-    task :create => :environment do
-      QC::Setup.create
-    end
+  desc "Setup queue_classic tables and functions in database"
+  task :create => :environment do
+    QC::Setup.create
+  end
 
-    desc "Remove queue_classic tables and functions from database."
-    task :drop => :environment do
-      QC::Setup.drop
-    end
+  desc "Remove queue_classic tables and functions from database."
+  task :drop => :environment do
+    QC::Setup.drop
   end
 end

data/lib/queue_classic/worker.rb

@@ -1,28 +1,19 @@
-require 'thread'
 require 'queue_classic'
 require 'queue_classic/queue'
+require 'queue_classic/conn_adapter'
 
 module QC
   class Worker
-    # Set this variable if you wish for
-    # the worker to fork a UNIX process for
-    # each locked job. Remember to re-establish
-    # any database connections. See the worker
-    # for more details.
-    FORK_WORKER = !ENV["QC_FORK_WORKER"].nil?
-    # The worker is capable of processing many jobs at a time.
-    # It uses FORK(2) to accomplish parallel processing. CONCURRENCY
-    # is used to set an uppoer bound on how many worker processes can
-    # run concurrently.
-    CONCURRENCY = Integer(ENV["QC_CONCURRENCY"] || 1)
 
-    attr_accessor :queue, :running
+    attr_accessor :queues, :running
     # In the case no arguments are passed to the initializer,
     # the defaults are pulled from the environment variables.
     def initialize(args={})
-      @fork_worker = args[:fork_worker] || FORK_WORKER || (CONCURRENCY > 1)
-      @limiter = SizedQueue.new(args[:concurrency] || CONCURRENCY)
-      @queue = args[:queue] || QC.default_queue
+      @fork_worker = args[:fork_worker] || QC::FORK_WORKER
+      @wait_interval = args[:wait_interval] || QC::WAIT_TIME
+      @conn_adapter = ConnAdapter.new(args[:connection])
+      @queues = setup_queues(@conn_adapter,
+        args[:q_name], args[:q_names], args[:top_bound])
       log(args.merge(:at => "worker_initialized"))
       @running = true
     end
@@ -47,26 +38,17 @@ module QC
     # Define setup_child to hook into the forking process.
     # Using setup_child is good for re-establishing database connections.
     def fork_and_work
-      # If the limiter is full, then we will block until space permits.
-      @limiter.enq(1)
-      Thread.new do
-        begin
-          cpid = fork {setup_child; work}
-          log(:at => :fork, :pid => cpid)
-          Process.wait(cpid)
-        ensure
-          # Once we are done with our work and our process has exited,
-          # we can allow another process to run.
-          @limiter.deq
-        end
-      end
+      cpid = fork {setup_child; work}
+      log(:at => :fork, :pid => cpid)
+      Process.wait(cpid)
     end
 
     # This method will lock a job & process the job.
     def work
-      if job = lock_job
+      queue, job = lock_job
+      if queue && job
         QC.log_yield(:at => "work", :job => job[:id]) do
-          process(job)
+          process(queue, job)
         end
       end
     end
@@ -78,23 +60,26 @@ module QC
       log(:at => "lock_job")
       job = nil
       while @running
-        break if job = @queue.lock
-        @queue.wait
+        @queues.each do |queue|
+          if job = queue.lock
+            return [queue, job]
+          end
+        end
+        @conn_adapter.wait(@wait_interval, *@queues.map {|q| q.name})
       end
-      job
     end
 
     # A job is processed by evaluating the target code.
     # Errors are delegated to the handle_failure method.
     # Also, this method will make the best attempt to delete the job
     # from the queue before returning.
-    def process(job)
+    def process(queue, job)
       begin
         call(job)
       rescue => e
         handle_failure(job, e)
       ensure
-        @queue.delete(job[:id])
+        queue.delete(job[:id])
         log(:at => "delete_job", :job => job[:id])
       end
     end
@@ -126,5 +111,18 @@ module QC
       QC.log(data)
     end
 
+    private
+
+    def setup_queues(adapter, q_name, q_names, top_bound)
+      name = q_name || QC::QUEUE
+      names = q_names || QC::QUEUES
+      names << name unless names.include?(name)
+      names.map do |name|
+        QC::Queue.new(name, top_bound).tap do |q|
+          q.conn_adapter = adapter
+        end
+      end
+    end
+
   end
 end

data/lib/queue_classic.rb

@@ -1,4 +1,36 @@
 module QC
+  # You can use the APP_NAME to query for
+  # postgres related process information in the
+  # pg_stat_activity table.
+  APP_NAME = ENV["QC_APP_NAME"] || "queue_classic"
+
+  # Number of seconds to block on the listen chanel for new jobs.
+  WAIT_TIME = (ENV["QC_LISTEN_TIME"] || 5).to_i
+
+  # Why do you want to change the table name?
+  # Just deal with the default OK?
+  # If you do want to change this, you will
+  # need to update the PL/pgSQL lock_head() function.
+  # Come on. Don't do it.... Just stick with the default.
+  TABLE_NAME = "queue_classic_jobs"
+
+  # Each row in the table will have a column that
+  # notes the queue. You can point your workers
+  # at different queues but only one at a time.
+  QUEUE = ENV["QUEUE"] || "default"
+  QUEUES = ENV["QUEUES"] || []
+
+  # Set this to 1 for strict FIFO.
+  # There is nothing special about 9....
+  TOP_BOUND = (ENV["QC_TOP_BOUND"] || 9).to_i
+
+  # Set this variable if you wish for
+  # the worker to fork a UNIX process for
+  # each locked job. Remember to re-establish
+  # any database connections. See the worker
+  # for more details.
+  FORK_WORKER = !ENV["QC_FORK_WORKER"].nil?
+
   # Defer method calls on the QC module to the
   # default queue. This facilitates QC.enqueue()
   def self.method_missing(sym, *args, &block)
@@ -15,7 +47,13 @@ module QC
   end
 
   def self.default_queue
-    @default_queue ||= Queue.new
+    @default_queue ||= begin
+      Queue.new(QUEUE)
+    end
+  end
+
+  def self.default_conn_adapter
+    @conn_adapter ||= ConnAdapter.new
   end
 
   def self.log_yield(data)
@@ -37,7 +75,7 @@ module QC
     if block_given?
       start = Time.now
       result = yield
-      data.merge(:elapsed => Integer((Time.now - start)*1000))
+      data.merge(:elapsed => Integer((Time.now - t0)*1000))
     end
     data.reduce(out=String.new) do |s, tup|
       s << [tup.first, tup.last].join("=") << " "

data/readme.md

@@ -1,6 +1,7 @@
 # queue_classic
 
-v2.2.1
+Stable: [v2.2.3](https://github.com/ryandotsmith/queue_classic/tree/v2.2.3)
+Latest: v3.0.0beta
 
 queue_classic provides a simple interface to a PostgreSQL-backed message queue. queue_classic specializes in concurrent locking and minimizing database load while providing a simple, intuitive developer experience. queue_classic assumes that you are already using PostgreSQL in your production environment and that adding another dependency (e.g. redis, beanstalkd, 0mq) is undesirable.
 
@@ -9,12 +10,13 @@ Features:
 * Leverage of PostgreSQL's listen/notify & row locking.
 * Support for multiple queues with heterogeneous workers.
 * JSON data format.
-* Concurrent job processing using forking workers.
-* [Reduced contention FIFO design](http://www.cs.tau.ac.il/~shanir/nir-pubs-web/Papers/Lock_Free.pdf).
+* Forking workers.
+* Workers can work multiple queues.
+* [Fuzzy-FIFO support](http://www.cs.tau.ac.il/~shanir/nir-pubs-web/Papers/Lock_Free.pdf).
 
 Contents:
 
-* [Documentation](http://rubydoc.info/gems/queue_classic/2.2.1/frames)
+* [Documentation](http://rubydoc.info/gems/queue_classic/2.2.3/frames)
 * [Usage](#usage)
 * [Setup](#setup)
 * [Configuration](#configuration)
@@ -56,17 +58,11 @@ p_queue.enqueue("Kernel.puts", ["hello", "world"])
 
 ### Working Jobs
 
-There are two ways to work jobs. The first approach is to use the Rake task. The second approach is to use a custom executable. Each approach provides a set of configuration options accessable through the processes' environment:
-
-* `$CONCURRENCY=1` - The number of child processes to run concurrently.
-* `$FORK_WORKER=false` - Fork on each job execution. Enabled if `$CONCURRENCY` > 1
-* `$QUEUE=default` - The name of the queue to process.
-* `$TOP_BOUND=9` - The section of the queue that is elgible for dequeue operations. Setting this value to 1 will ensure a strict FIFO ordering.
+There are two ways to work jobs. The first approach is to use the Rake task. The second approach is to use a custom executable.
 
 #### Rake Task
 
-Require queue_classic in your Rakefile. If you are using Rails, the tasks will
-be loaded automatically.
+Require queue_classic in your Rakefile.
 
 ```ruby
 require 'queue_classic'
@@ -79,10 +75,16 @@ Start the worker via the Rakefile.
 $ bundle exec rake qc:work
 ```
 
-Setup a worker to work a non-default queue while processing 4 jobs at a time.
+Setup a worker to work a non-default queue.
+
+```bash
+$ QUEUE="priority_queue" bundle exec rake qc:work
+```
+
+Setup a worker to work multiple queues.
 
 ```bash
-$ CONCURRENCY=4 QUEUE="priority_queue" bundle exec rake qc:work
+$ QUEUE="priority_queue, secondary_queue" bundle exec rake qc:work
 ```
 
 #### Custom Worker
@@ -135,7 +137,7 @@ Declare dependencies in Gemfile.
 
 ```ruby
 source "http://rubygems.org"
-gem "queue_classic", "2.2.1"
+gem "queue_classic", "2.2.3"
 ```
 
 By default, queue_classic will use the QC_DATABASE_URL falling back on DATABASE_URL. The URL must be in the following format: `postgres://username:password@localhost/database_name`.  If you use Heroku's PostgreSQL service, this will already be set. If you don't want to set this variable, you can set the connection in an initializer. **QueueClassic will maintain its own connection to the database.** This may double the number of connections to your database. Set QC::Conn.connection to share the connection between Rails & QueueClassic

data/test/benchmark_test.rb

@@ -1,5 +1,4 @@
 require File.expand_path("../helper.rb", __FILE__)
-Thread.abort_on_exception = true
 
 if ENV["QC_BENCHMARK"]
   class BenchmarkTest < QCTest
@@ -7,26 +6,31 @@ if ENV["QC_BENCHMARK"]
     def test_enqueue
       n = 10_000
       start = Time.now
-      n.times {QC.enqueue("1.odd?")}
+      n.times do
+        QC.enqueue("1.odd?", [])
+      end
       assert_equal(n, QC.count)
+
       elapsed = Time.now - start
       assert_in_delta(4, elapsed, 1)
     end
 
     def test_dequeue
-      queue = QC::Queue.new
-      worker = QC::Worker.new(:concurrency => 4, :queue => queue)
-      queue.delete_all
-      n = 20
-
-      n.times {queue.enqueue("puts", "hello")}
-      assert_equal(n, queue.count)
+      worker = QC::Worker.new
+      worker.running = true
+      n = 10_000
+      n.times do
+        QC.enqueue("1.odd?", [])
+      end
+      assert_equal(n, QC.count)
 
       start = Time.now
-      n.times.map {worker.fork_and_work}.map(&:join)
-
+      n.times do
+        worker.work
+      end
       elapsed = Time.now - start
-      assert_equal(0, queue.count)
+
+      assert_equal(0, QC.count)
       assert_in_delta(10, elapsed, 3)
     end
 

data/test/helper.rb

@@ -9,10 +9,19 @@ require "minitest/autorun"
 
 class QCTest < Minitest::Test
 
+  def setup
+    init_db
+  end
+
+  def teardown
+    QC.delete_all
+  end
+
   def init_db
-    c = QC::Conn.new
-    QC::Setup.drop(c)
-    QC::Setup.create(c)
+    c = QC::ConnAdapter.new
+    c.execute("SET client_min_messages TO 'warning'")
+    QC::Setup.drop(c.connection)
+    QC::Setup.create(c.connection)
     c.execute(File.read('./test/helper.sql'))
     c.disconnect
   end

data/test/queue_test.rb

@@ -2,13 +2,7 @@ require File.expand_path("../helper.rb", __FILE__)
 
 class QueueTest < QCTest
 
-  def setup
-    init_db
-  end
-
-  def teardown
-    QC.conn.disconnect
-  end
+  ResetError = Class.new(PGError)
 
   def test_enqueue
     QC.enqueue("Klass.method")
@@ -52,8 +46,8 @@ class QueueTest < QCTest
   end
 
   def test_delete_all_by_queue_name
-    p_queue = QC::Queue.new(:name => "priority_queue")
-    s_queue = QC::Queue.new(:name => "secondary_queue")
+    p_queue = QC::Queue.new("priority_queue")
+    s_queue = QC::Queue.new("secondary_queue")
     p_queue.enqueue("Klass.method")
     s_queue.enqueue("Klass.method")
     assert_equal(1, p_queue.count)
@@ -61,19 +55,27 @@ class QueueTest < QCTest
     p_queue.delete_all
     assert_equal(0, p_queue.count)
     assert_equal(1, s_queue.count)
-  ensure
-    p_queue.conn.disconnect
-    s_queue.conn.disconnect
   end
 
   def test_queue_instance
-    queue = QC::Queue.new(:name => "queue_classic_jobs")
+    queue = QC::Queue.new("queue_classic_jobs")
     queue.enqueue("Klass.method")
     assert_equal(1, queue.count)
     queue.delete(queue.lock[:id])
     assert_equal(0, queue.count)
-  ensure
-    queue.conn.disconnect
+  end
+
+  def test_repair_after_error
+    queue = QC::Queue.new("queue_classic_jobs")
+    queue.conn_adapter = QC::ConnAdapter.new
+    queue.enqueue("Klass.method")
+    assert_equal(1, queue.count)
+    conn = queue.conn_adapter.connection
+    def conn.exec(*args); raise(PGError); end
+    def conn.reset(*args); raise(ResetError)  end
+    # We ensure that the reset method is called on the connection.
+    assert_raises(PG::Error, ResetError) {queue.enqueue("Klass.other_method")}
+    queue.conn_adapter.disconnect
   end
 
   def test_custom_default_queue
@@ -97,17 +99,16 @@ class QueueTest < QCTest
   end
 
   def test_enqueue_triggers_notify
-    c = QC::Conn.new
-    c.execute('LISTEN "' + QC::Queue::QUEUE_NAME + '"')
-    c.send(:drain_notify)
-    msgs = c.send(:wait_for_notify, 0.25)
+    adapter = QC.default_conn_adapter
+    adapter.execute('LISTEN "' + QC::QUEUE + '"')
+    adapter.send(:drain_notify)
+
+    msgs = adapter.send(:wait_for_notify, 0.25)
     assert_equal(0, msgs.length)
 
     QC.enqueue("Klass.method")
-    msgs = c.send(:wait_for_notify, 0.25)
+    msgs = adapter.send(:wait_for_notify, 0.25)
     assert_equal(1, msgs.length)
-  ensure
-    c.disconnect
   end
 
 end

data/test/worker_test.rb

@@ -14,26 +14,19 @@ end
 class TestWorker < QC::Worker
   attr_accessor :failed_count
 
-  def initialize(*args)
-    super(*args)
+  def initialize(args={})
+    super(args.merge(:connection => QC.default_conn_adapter.connection))
     @failed_count = 0
   end
 
   def handle_failure(job,e)
     @failed_count += 1
+    super
   end
 end
 
 class WorkerTest < QCTest
 
-  def setup
-    init_db
-  end
-
-  def teardown
-    QC.conn.disconnect
-  end
-
   def test_work
     QC.enqueue("TestObject.no_args")
     worker = TestWorker.new
@@ -53,7 +46,7 @@ class WorkerTest < QCTest
   def test_failed_job_is_logged
     output = capture_debug_output do
       QC.enqueue("TestObject.not_a_method")
-      QC::Worker.new.work
+      TestWorker.new.work
     end
     expected_output = /lib=queue-classic at=handle_failure job={:id=>"\d+", :method=>"TestObject.not_a_method", :args=>\[\]} error=#<NoMethodError: undefined method `not_a_method' for TestObject:Module>/
     assert_match(expected_output, output, "=== debug output ===\n #{output}")
@@ -102,40 +95,76 @@ class WorkerTest < QCTest
   end
 
   def test_work_custom_queue
-    p_queue = QC::Queue.new(:name=> "priority_queue")
+    p_queue = QC::Queue.new("priority_queue")
     p_queue.enqueue("TestObject.two_args", "1", 2)
-    worker = TestWorker.new(:queue => p_queue)
+    worker = TestWorker.new(q_name: "priority_queue")
     r = worker.work
     assert_equal(["1", 2], r)
     assert_equal(0, worker.failed_count)
-    worker.stop
-    p_queue.conn.disconnect
   end
 
   def test_worker_listens_on_chan
-    p_queue = QC::Queue.new(:name => "priority_queue")
+    p_queue = QC::Queue.new("priority_queue")
+    # Use a new connection because the default connection
+    # will be locked by the sleeping worker.
+    p_queue.conn_adapter = QC::ConnAdapter.new
+    # The wait interval is extreme to demonstrate
+    # that the worker is in fact being activated by a NOTIFY.
+    worker = TestWorker.new(:q_name => "priority_queue", :wait_interval => 100)
+    t = Thread.new do
+      r = worker.work
+      assert_equal(["1", 2], r)
+      assert_equal(0, worker.failed_count)
+    end
+    sleep(0.5) #Give the thread some time to start the worker.
     p_queue.enqueue("TestObject.two_args", "1", 2)
-    worker = TestWorker.new(
-      :queue => p_queue,
-      :listening_worker => true)
-    r = worker.work
-    assert_equal(["1", 2], r)
-    assert_equal(0, worker.failed_count)
-    worker.stop
-    p_queue.conn.disconnect
+    p_queue.conn_adapter.disconnect
+    t.join
   end
 
   def test_worker_ueses_one_conn
     QC.enqueue("TestObject.no_args")
     worker = TestWorker.new
     worker.work
-    s = "SELECT * from pg_stat_activity where datname=current_database()"
-    s += " and application_name = '#{QC::Conn::APP_NAME}'"
-    res = QC.conn.execute(s)
-    num_conns = res.length if res.class == Array
-    num_conns = 1 if res.class == Hash
-    assert_equal(1, num_conns,
-      "Multiple connections found -- are there open connections to" +
-        " #{QC::Conf.db_url} in other terminals?\n res=#{res}")
+    assert_equal(
+      1,
+      QC.default_conn_adapter.execute("SELECT count(*) from pg_stat_activity where datname = current_database()")["count"].to_i,
+      "Multiple connections found -- are there open connections to #{ QC.default_conn_adapter.send(:db_url) } in other terminals?"
+    )
+  end
+
+  def test_worker_can_work_multiple_queues
+    p_queue = QC::Queue.new("priority_queue")
+    p_queue.enqueue("TestObject.two_args", "1", 2)
+
+    s_queue = QC::Queue.new("secondary_queue")
+    s_queue.enqueue("TestObject.two_args", "1", 2)
+
+    worker = TestWorker.new(:q_names => ["priority_queue", "secondary_queue"])
+
+    2.times do
+      r = worker.work
+      assert_equal(["1", 2], r)
+      assert_equal(0, worker.failed_count)
+    end
+  end
+
+  def test_worker_works_multiple_queue_left_to_right
+    l_queue = QC::Queue.new("left_queue")
+    r_queue = QC::Queue.new("right_queue")
+
+    3.times { l_queue.enqueue("TestObject.two_args", "1", 2) }
+    3.times { r_queue.enqueue("TestObject.two_args", "1", 2) }
+
+    worker = TestWorker.new(:q_names => ["left_queue", "right_queue"])
+
+    worker.work
+    assert_equal(2, l_queue.count)
+    assert_equal(3, r_queue.count)
+
+    worker.work
+    assert_equal(1, l_queue.count)
+    assert_equal(3, r_queue.count)
   end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: queue_classic
 version: !ruby/object:Gem::Version
-  version: 2.3.0beta
+  version: 3.0.0beta
 platform: ruby
 authors:
 - Ryan Smith (♠ ace hacker)
@@ -16,18 +16,18 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.16.0
+        version: 0.17.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.16.0
+        version: 0.17.0
 description: queue_classic is a queueing library for Ruby apps. (Rails, Sinatra, Etc...)
   queue_classic features asynchronous job polling, database maintained locks and no
   ridiculous dependencies. As a matter of fact, queue_classic only requires pg.
-email: ryan@heroku.com
+email: r@32k.io
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -38,8 +38,7 @@ files:
 - sql/drop_ddl.sql
 - lib/generators/queue_classic/install_generator.rb
 - lib/generators/queue_classic/templates/add_queue_classic.rb
-- lib/queue_classic/conf.rb
-- lib/queue_classic/conn.rb
+- lib/queue_classic/conn_adapter.rb
 - lib/queue_classic/queue.rb
 - lib/queue_classic/railtie.rb
 - lib/queue_classic/setup.rb
@@ -47,7 +46,6 @@ files:
 - lib/queue_classic/worker.rb
 - lib/queue_classic.rb
 - test/benchmark_test.rb
-- test/conf_test.rb
 - test/helper.rb
 - test/queue_test.rb
 - test/worker_test.rb
@@ -74,9 +72,8 @@ rubyforge_project:
 rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
-summary: postgres backed queue
+summary: Simple, efficient worker queue for Ruby & PostgreSQL.
 test_files:
 - test/benchmark_test.rb
-- test/conf_test.rb
 - test/queue_test.rb
 - test/worker_test.rb

data/lib/queue_classic/conf.rb

@@ -1,33 +0,0 @@
-require 'uri'
-
-module QC
-  module Conf
-
-    def self.env(k); ENV[k]; end
-    def self.env!(k); env(k) || raise("Must set #{k}."); end
-
-    def self.debug?
-      !env('DEBUG').nil?
-    end
-
-    def self.db_url
-      url = env("QC_DATABASE_URL") ||
-            env("DATABASE_URL")    ||
-            raise(ArgumentError, "Must set QC_DATABASE_URL or DATABASE_URL.")
-      URI.parse(url)
-    end
-
-    def self.normalized_db_url(url=nil)
-      url ||= db_url
-      host = url.host
-      host = host.gsub(/%2F/i, '/') if host
-      [host, # host or percent-encoded socket path
-       url.port || 5432,
-       nil, '', #opts, tty
-       url.path.gsub("/",""), # database name
-       url.user,
-       url.password]
-    end
-
-  end
-end

data/lib/queue_classic/conn.rb

@@ -1,106 +0,0 @@
-require 'queue_classic/conf'
-require 'thread'
-require 'uri'
-require 'pg'
-
-module QC
-  class Conn
-    # Number of seconds to block on the listen chanel for new jobs.
-    WAIT_TIME = (ENV["QC_LISTEN_TIME"] || 5).to_i
-    # You can use the APP_NAME to query for
-    # postgres related process information in the
-    # pg_stat_activity table.
-    APP_NAME = ENV["QC_APP_NAME"] || "queue_classic"
-
-
-    def self.connect
-      QC.log(:at => "establish_conn")
-      conn = PGconn.connect(*Conf.normalized_db_url)
-      if conn.status != PGconn::CONNECTION_OK
-        log(:error => conn.error)
-      end
-      if !Conf.debug?
-        conn.exec("SET client_min_messages TO 'warning'")
-      end
-      conn.exec("SET application_name = '#{APP_NAME}'")
-      conn
-    end
-
-    def initialize
-      @c = self.class.connect
-      @max_attempts = 2
-    end
-
-    def execute(stmt, *params)
-      QC.log(:measure => "conn.exec", :sql => stmt.inspect) do
-        with_retry(@max_attempts) do
-          params = nil if params.empty?
-          r = @c.exec(stmt, params)
-          result = []
-          r.each {|t| result << t}
-          result.length > 1 ? result : result.pop
-        end
-      end
-    end
-
-    def wait(chan)
-      with_retry(@max_attempts) do
-        execute('LISTEN "' + chan + '"')
-        wait_for_notify(WAIT_TIME)
-        execute('UNLISTEN "' + chan + '"')
-        drain_notify
-      end
-    end
-
-    def reconnect
-      disconnect
-      @c = self.class.connect
-    end
-
-    def disconnect
-      begin @c && @c.finish
-      ensure @c = nil
-      end
-    end
-
-    def abort_open_transaction
-      if @c.transaction_status != PGconn::PQTRANS_IDLE
-        @c.exec('ROLLBACK')
-      end
-    end
-
-    private
-
-    def with_retry(n)
-      completed = false
-      attempts = 0
-      result = nil
-      last_error = nil
-      until completed || attempts == n
-        attempts += 1
-        begin
-          result = yield
-          completed = true
-        rescue => e
-          QC.log(:error => e.class, :at => 'conn-retry', :attempts => attempts)
-          last_error = e
-          reconnect
-        end
-      end
-      completed ? result : raise(last_error)
-    end
-
-    def wait_for_notify(t)
-      Array.new.tap do |msgs|
-        @c.wait_for_notify(t) {|event, pid, msg| msgs << msg}
-      end
-    end
-
-    def drain_notify
-      until @c.notifies.nil?
-        QC.log(:at => "drain_notifications")
-      end
-    end
-
-  end
-end

data/test/conf_test.rb

@@ -1,26 +0,0 @@
-require File.expand_path("../helper.rb", __FILE__)
-
-class ConfTest < QCTest
-
-  def setup
-    init_db
-  end
-
-  def test_extracts_the_segemnts_to_connect
-    database_url = "postgres://ryan:secret@localhost:1234/application_db"
-    normalized = QC::Conf.normalized_db_url(URI.parse(database_url))
-    assert_equal ["localhost",
-                  1234,
-                  nil, "",
-                  "application_db",
-                  "ryan",
-                  "secret"], normalized
-  end
-
-  def test_regression_database_url_without_host
-    database_url = "postgres:///my_db"
-    normalized = QC::Conf.normalized_db_url(URI.parse(database_url))
-    assert_equal [nil, 5432, nil, "", "my_db", nil, nil], normalized
-  end
-
-end