queue_classic 2.0.5 → 2.1.0

data/lib/queue_classic/conn.rb

@@ -1,46 +1,51 @@
+require 'thread'
+
 module QC
   module Conn
     extend self
+    @exec_mutex = Mutex.new
 
     def execute(stmt, *params)
-      log(:level => :debug, :action => "exec_sql", :sql => stmt.inspect)
-      begin
-        params = nil if params.empty?
-        r = connection.exec(stmt, params)
-        result = []
-        r.each {|t| result << t}
-        result.length > 1 ? result : result.pop
-      rescue PGError => e
-        log(:error => e.inspect)
-        disconnect
-        raise
+      @exec_mutex.synchronize do
+        log(:at => "exec_sql", :sql => stmt.inspect)
+        begin
+          params = nil if params.empty?
+          r = connection.exec(stmt, params)
+          result = []
+          r.each {|t| result << t}
+          result.length > 1 ? result : result.pop
+        rescue PGError => e
+          log(:error => e.inspect)
+          disconnect
+          raise
+        end
       end
     end
 
     def notify(chan)
-      log(:level => :debug, :action => "NOTIFY")
+      log(:at => "NOTIFY")
       execute('NOTIFY "' + chan + '"') #quotes matter
     end
 
     def listen(chan)
-      log(:level => :debug, :action => "LISTEN")
+      log(:at => "LISTEN")
       execute('LISTEN "' + chan + '"') #quotes matter
     end
 
     def unlisten(chan)
-      log(:level => :debug, :action => "UNLISTEN")
+      log(:at => "UNLISTEN")
       execute('UNLISTEN "' + chan + '"') #quotes matter
     end
 
     def drain_notify
       until connection.notifies.nil?
-        log(:level => :debug, :action => "drain_notifications")
+        log(:at => "drain_notifications")
       end
     end
 
     def wait_for_notify(t)
       connection.wait_for_notify(t) do |event, pid, msg|
-        log(:level => :debug, :action => "received_notification")
+        log(:at => "received_notification")
       end
     end
 
@@ -65,22 +70,21 @@ module QC
 
     def connection=(connection)
       unless connection.instance_of? PG::Connection
-        raise(
-          ArgumentError,
-          "connection must be an instance of PG::Connection, but was #{connection.class}"
-        )
+        c = connection.class
+        err = "connection must be an instance of PG::Connection, but was #{c}"
+        raise(ArgumentError, err)
       end
       @connection = connection
     end
 
     def disconnect
-      connection.finish
-    ensure
-      @connection = nil
+      begin connection.finish
+      ensure @connection = nil
+      end
     end
 
     def connect
-      log(:level => :debug, :action => "establish_conn")
+      log(:at => "establish_conn")
       conn = PGconn.connect(
         db_url.host,
         db_url.port || 5432,
@@ -90,7 +94,7 @@ module QC
         db_url.password
       )
       if conn.status != PGconn::CONNECTION_OK
-        log(:level => :error, :message => conn.error)
+        log(:error => conn.error)
       end
       conn
     end

data/lib/queue_classic/queries.rb

@@ -4,7 +4,7 @@ module QC
 
     def insert(q_name, method, args, chan=nil)
       QC.log_yield(:action => "insert_job") do
-        s = "INSERT INTO #{TABLE_NAME} (q_name, method, args) VALUES ($1, $2, $3)"
+        s="INSERT INTO #{TABLE_NAME} (q_name, method, args) VALUES ($1, $2, $3)"
         res = Conn.execute(s, q_name, method, OkJson.encode(args))
         Conn.notify(chan) if chan
       end

data/lib/queue_classic/queue.rb

@@ -2,7 +2,6 @@ module QC
   class Queue
 
     attr_reader :name, :chan
-
     def initialize(name, notify=QC::LISTENING_WORKER)
       @name = name
       @chan = @name if notify

data/lib/queue_classic/tasks.rb

@@ -8,7 +8,10 @@ end
 namespace :qc do
   desc "Start a new worker for the (default or $QUEUE) queue"
   task :work  => :environment do
-    QC::Worker.new.start
+    trap('INT') {exit}
+    trap('TERM') {@worker.stop}
+    @worker = QC::Worker.new
+    @worker.start
   end
 
   desc "Returns the number of jobs in the (default or QUEUE) queue"

data/lib/queue_classic/worker.rb

@@ -2,126 +2,97 @@ module QC
   class Worker
 
     attr_reader :queue
-
-    def initialize(*args)
-      if args.length == 5
-        q_name, top_bound, fork_worker, listening_worker, max_attempts = *args
-      elsif args.length <= 1
-        opts = args.first || {}
-        q_name           = opts[:q_name]           || QC::QUEUE
-        top_bound        = opts[:top_bound]        || QC::TOP_BOUND
-        fork_worker      = opts[:fork_worker]      || QC::FORK_WORKER
-        listening_worker = opts[:listening_worker] || QC::LISTENING_WORKER
-        max_attempts     = opts[:max_attempts]     || QC::MAX_LOCK_ATTEMPTS
-      else
-        raise ArgumentError, 'wrong number of arguments (expected no args, an options hash, or 5 separate args)'
-      end
+    # In the case no arguments are passed to the initializer,
+    # the defaults are pulled from the environment variables.
+    def initialize(args={})
+      @q_name           = args[:q_name]           ||= QC::QUEUE
+      @top_bound        = args[:top_bound]        ||= QC::TOP_BOUND
+      @fork_worker      = args[:fork_worker]      ||= QC::FORK_WORKER
+      @listening_worker = args[:listening_worker] ||= QC::LISTENING_WORKER
+      @max_attempts     = args[:max_attempts]     ||= QC::MAX_LOCK_ATTEMPTS
 
       @running = true
-      @queue = Queue.new(q_name, listening_worker)
-      @top_bound = top_bound
-      @fork_worker = fork_worker
-      @listening_worker = listening_worker
-      @max_attempts = max_attempts
-      handle_signals
-
-      log(
-        :level => :debug,
-        :action => "worker_initialized",
-        :queue => q_name,
-        :top_bound => top_bound,
-        :fork_worker => fork_worker,
-        :listening_worker => listening_worker,
-        :max_attempts => max_attempts
-      )
-    end
-
-    def running?
-      @running
-    end
-
-    def fork_worker?
-      @fork_worker
-    end
-
-    def can_listen?
-      @listening_worker
+      @queue = Queue.new(@q_name, @listening_worker)
+      log(args.merge(:at => "worker_initialized"))
     end
 
-    def handle_signals
-      %W(INT TERM).each do |sig|
-        trap(sig) do
-          if running?
-            @running = false
-            log(:level => :debug, :action => "handle_signal", :running => @running)
-          else
-            raise Interrupt
-          end
-        end
+    # Start a loop and work jobs indefinitely.
+    # Call this method to start the worker.
+    # This is the easiest way to start working jobs.
+    def start
+      while @running
+        @fork_worker ? fork_and_work : work
       end
     end
 
-    # This method should be overriden if
-    # your worker is forking and you need to
-    # re-establish database connectoins
-    def setup_child
-    end
-
-    def start
-      while running?
-        if fork_worker?
-          fork_and_work
-        else
-          work
-        end
-      end
+    # Call this method to stop the worker.
+    # The worker may not stop immediately if the worker
+    # is sleeping.
+    def stop
+      @running = false
     end
 
+    # This method will tell the ruby process to FORK.
+    # Define setup_child to hook into the forking process.
+    # Using setup_child is good for re-establishing database connections.
     def fork_and_work
-      @cpid = fork { setup_child; work }
-      log(:level => :debug, :action => :fork, :pid => @cpid)
+      @cpid = fork {setup_child; work}
+      log(:at => :fork, :pid => @cpid)
       Process.wait(@cpid)
     end
 
+    # This method will lock a job & evaluate the code defined by the job.
+    # Also, this method will make the best attempt to delete the job
+    # from the queue before returning.
     def work
       if job = lock_job
-        QC.log_yield(:level => :info, :action => "work_job", :job => job[:id]) do
+        QC.log_yield(:at => "work", :job => job[:id]) do
           begin
             call(job)
-          rescue Object => e
-            log(:level => :debug, :action => "failed_work", :job => job[:id], :error => e.inspect)
+          rescue => e
             handle_failure(job, e)
           ensure
             @queue.delete(job[:id])
-            log(:level => :debug, :action => "delete_job", :job => job[:id])
+            log(:at => "delete_job", :job => job[:id])
           end
         end
       end
     end
 
+    # lock_job will attempt to lock a job in the queue's table. It uses an
+    # exponential backoff in the event that a job was not locked. This method
+    # will return a hash when a job is obtained.
+    #
+    # This method will terminate early if the stop method is called or
+    # @max_attempts has been reached.
+    #
+    # It is important that callers delete the job when finished.
+    # *@queue.delete(job[:id])*
     def lock_job
-      log(:level => :debug, :action => "lock_job")
+      log(:at => "lock_job")
       attempts = 0
       job = nil
-      until !running? || job
+      until !@running || job
         job = @queue.lock(@top_bound)
         if job.nil?
-          log(:level => :debug, :action => "failed_lock", :attempts => attempts)
+          log(:at => "failed_lock", :attempts => attempts)
           if attempts < @max_attempts
-            seconds = 2**attempts
-            wait(seconds)
+            wait(2**attempts)
             attempts += 1
             next
           else
             break
           end
         else
-          log(:level => :debug, :action => "finished_lock", :job => job[:id])
+          log(:at => "finished_lock", :job => job[:id])
         end
       end
       job
     end
 
+    # Each job includes a method column. We will use ruby's eval
+    # to grab the ruby object from memory. We send the method to
+    # the object and pass the args.
     def call(job)
       args = job[:args]
       klass = eval(job[:method].split(".").first)
@@ -129,27 +100,35 @@ module QC
       klass.send(message, *args)
     end
 
+    # If @listening_worker is set, the worker will use the database
+    # to sleep. The database approach preferred over a syscall since
+    # the database will break the sleep when new jobs are inserted into
+    # the queue.
     def wait(t)
-      if can_listen?
-        log(:level => :debug, :action => "listen_wait", :wait => t)
+      if @listening_worker
+        log(:at => "listen_wait", :wait => t)
         Conn.listen(@queue.chan)
         Conn.wait_for_notify(t)
         Conn.unlisten(@queue.chan)
         Conn.drain_notify
-        log(:level => :debug, :action => "finished_listening")
+        log(:at => "finished_listening")
       else
-        log(:level => :debug, :action => "sleep_wait", :wait => t)
+        log(:at => "sleep_wait", :wait => t)
         Kernel.sleep(t)
       end
     end
 
-    #override this method to do whatever you want
+    # This method will be called when an exception
+    # is raised during the execution of the job.
     def handle_failure(job,e)
-      puts "!"
-      puts "! \t FAIL"
-      puts "! \t \t #{job.inspect}"
-      puts "! \t \t #{e.inspect}"
-      puts "!"
+      log(:at => "handle_failure")
+    end
+
+    # This method should be overriden if
+    # your worker is forking and you need to
+    # re-establish database connections
+    def setup_child
+      log(:at => "setup_child")
     end
 
     def log(data)

data/readme.md

@@ -1,175 +1,34 @@
 # queue_classic
 
-v2.0.4
+v2.1.0
 
-queue_classic provides PostgreSQL-backed queueing focused on concurrent job
-locking and minimizing database load while providing a simple, intuitive user
-experience.
+queue_classic provides a simple interface to a PostgreSQL-backed message queue. queue_classic specializes in concurrent locking and minimizing database load while providing a simple, intuitive developer experience. queue_classic assumes that you are already using PostgreSQL in your production environment and that adding another dependency (e.g. redis, beanstalkd, 0mq) is undesirable.
 
-queue_classic features:
+Features:
 
-* Support for multiple queues with heterogeneous workers
-* Utilization of Postgres' PUB/SUB
-* JSON encoding
-* Forking workers
-* Postgres' rock-solid locking mechanism
-* Fuzzy-FIFO support [academic paper](http://www.cs.tau.ac.il/~shanir/nir-pubs-web/Papers/Lock_Free.pdf)
-* Instrumentation via log output
-* Long term support
+* Leverage of PostgreSQL's listen/notify & row locking.
+* Support for multiple queues with heterogeneous workers.
+* JSON data format.
+* Forking workers.
+* [Fuzzy-FIFO support](http://www.cs.tau.ac.il/~shanir/nir-pubs-web/Papers/Lock_Free.pdf)
 
-## Proven
+Contents:
 
-queue_classic was designed out of the necessity for a fast, reliable, low
-maintenance message queue.  It was built upon PostgreSQL to avoid the necessity
-of adding redis or 0MQ services to my applications. It was designed to be
-simple, with a small API and very few features. For a simple mechanism to
-distribute jobs to worker processes, especially if you are already running
-PostgreSQL, queue_classic is exactly what you should be using. If you need
-more advanced queueing features, you should investigate 0MQ, rabbitmq, or redis.
-
-### Heroku Postgres
-
-The Heroku Postgres team uses queue_classic to monitor the health of
-customer databases, processing 200 jobs per second using a [fugu](https://postgres.heroku.com/pricing)
-database. They chose queue_classic because of its simplicity and reliability.
-
-### Cloudapp
-
-Larry uses queue_classic to deliver cloudapp's push notifications and collect
-file meta-data from S3, processing nearly 14 jobs per second.
-
-```
-I haven't even touched QC since setting it up.
-The best queue is the one you don't have to hand hold.
-
--- Larry Marburger
-```
-
-## Setup
-
-In addition to installing the rubygem, you will need to prepare your database.
-Database preparation includes creating a table and loading PL/pgSQL functions.
-You can issue the database preparation commands using **PSQL(1)** or place them in a
-database migration.
-
-### Quick Start
-
-```bash
-$ gem install queue_classic
-$ createdb queue_classic_test
-$ export QC_DATABASE_URL="postgres://username:password@localhost/queue_classic_test"
-$ ruby -r queue_classic -e "QC::Setup.create"
-$ ruby -r queue_classic -e "QC.enqueue('Kernel.puts', 'hello world')"
-$ ruby -r queue_classic -e "QC::Worker.new.work"
-```
-
-### Ruby on Rails Setup
-
-**Gemfile**
-
-```ruby
-source :rubygems
-gem "queue_classic", "2.0.1"
-```
-
-**Rakefile**
-
-```ruby
-require "queue_classic"
-require "queue_classic/tasks"
-```
-
-**config/initializers/queue_classic.rb**
-
-```ruby
-# queue_classic will by default look for an environment variable DATABASE_URL
-# or QC_DATABASE_URL for a connection string in the format
-# "postgres://username:password@localhost/database_name".  If you use Heroku,
-# this will already be set.
-#
-# If you don't want to set this variable, you can set the connection in an
-# initializer.
-QC::Conn.connection = ActiveRecord::Base.connection.raw_connection
-```
-
-queue_classic requires a database table and a PL/pgSQL function to be loaded
-into your database. You can load the table and the function by running a migration
-or using a rake task.
-
-**db/migrations/add_queue_classic.rb**
-
-If you use the migration, and you wish to use commands that reset the database
-from the stored schema (e.g. `rake db:reset`), your application must be
-configured with `config.active_record.schema_format = :sql` in
-`config/application.rb`.  If you don't do this, the PL/pgSQL function that
-queue_classic creates will be lost when you reset the database.
-
-```ruby
-require 'queue_classic'
-
-class AddQueueClassic < ActiveRecord::Migration
-
-  def self.up
-    QC::Setup.create
-  end
-
-  def self.down
-    QC::Setup.drop
-  end
-
-end
-```
-
-**Rake Task**
-
-```bash
-# Creating the table and functions
-$ bundle exec rake qc:create
-
-# Dropping the table and functions
-$ bundle exec rake qc:drop
-```
-
-
-### Sequel Setup
-
-**db/migrations/1_add_queue_classic.rb**
-
-```ruby
-require 'queue_classic'
-
-Sequel.migration do
-  up {QC::Setup.create}
-  down {QC::Setup.drop}
-end
-```
-
-## Configure
-
-All configuration takes place in the form of environment vars.
-See [queue_classic.rb](https://github.com/ryandotsmith/queue_classic/blob/master/lib/queue_classic.rb#L23-62)
-for a list of options.
+* [Documentation](http://rubydoc.info/gems/queue_classic/2.1.0/frames)
+* [Usage](#usage)
+* [Setup](#setup)
+* [Configuration](#configuration)
+* [Hacking](#hacking-on-queue_classic)
+* [License](#license)
 
 ## Usage
 
-Users of queue_classic will be producing jobs (enqueue) or
-consuming jobs (lock then delete).
+There are 2 ways to use queue_classic.
 
-### Producer
+* Producing Jobs
+* Working Jobs
 
-You certainly don't need the queue_classic rubygem to put a job in the queue.
-
-```bash
-$ psql queue_classic_test -c "INSERT INTO queue_classic_jobs (q_name, method, args) VALUES ('default', 'Kernel.puts', '[\"hello world\"]');"
-```
-
-However, the rubygem will take care of converting your args to JSON and it will also dispatch
-PUB/SUB notifications if the feature is enabled. It will also manage a connection to the database
-that is independent of any other connection you may have in your application. Note: If your
-queue table is in your application's database then your application's process will have 2 connections
-to the database; one for your application and another for queue_classic.
-
-The Ruby API for producing jobs is pretty simple:
+### Producing Jobs
 
 ```ruby
 # This method has no arguments.
@@ -186,380 +45,151 @@ QC.enqueue("Kernel.puts", {"hello" => "world"})
 
 # This method has an array argument.
 QC.enqueue("Kernel.puts", ["hello", "world"])
+
+# This method uses a non-default queue.
+p_queue = QC::Queue.new(q_name: "priority_queue")
+p_queue.enqueue("Kernel.puts", ["hello", "world"])
 ```
 
-The basic idea is that all arguments should be easily encoded to json. OkJson
-is used to encode the arguments, so the arguments can be anything that OkJson can encode.
+QueueClassic uses [OkJson](https://github.com/kr/okjson) to encode the job's payload.
 
 ```ruby
-# Won't work!
-OkJson.encode({:test => "test"})
-
 # OK
 OkJson.encode({"test" => "test"})
-```
-
-To see more information on usage, take a look at the test files in the source code. Also,
-read up on [OkJson](https://github.com/kr/okjson)
-
-#### Multiple Queues
-
-The table containing the jobs has a column named *q_name*. This column
-is the abstraction queue_classic uses to represent multiple queues. This allows
-the programmer to place triggers and indexes on distinct queues.
-
-```ruby
-# attach to the priority_queue. this will insert
-# jobs with the column q_name = 'priority_queue'
-p_queue = QC::Queue.new("priority_queue")
-
-# This method has no arguments.
-p_queue.enqueue("Time.now")
-
-# This method has 1 argument.
-p_queue.enqueue("Kernel.puts", "hello world")
-
-# This method has 2 arguments.
-p_queue.enqueue("Kernel.printf", "hello %s", "world")
 
-# This method has a hash argument.
-p_queue.enqueue("Kernel.puts", {"hello" => "world"})
-
-# This method has an array argument.
-p_queue.enqueue("Kernel.puts", ["hello", "world"])
+# NOT OK
+OkJson.encode({:test => "test"})
 ```
 
-This code example shows how to produce jobs into a custom queue,
-to consume jobs from the custom queue be sure and set the `$QUEUE`
-var to the q_name in the worker's UNIX environment.
-
-### Consumer
+### Working Jobs
 
-There are several approaches to working jobs. The first is to include
-a task file provided by queue_classic and the other approach is to
-write a custom bin file.
+There are two ways to work jobs. The first approach is to use the Rake task. The second approach is to use a custom executable.
 
 #### Rake Task
 
-Be sure to include `queue_classic` and `queue_classic/tasks`
-in your primary Rakefile.
+Require queue_classic in your Rakefile.
+
+```ruby
+require 'queue_classic'
+require 'queue_classic/tasks'
+```
 
-To work jobs from the default queue:
+Start the worker via the Rakefile.
 
 ```bash
 $ bundle exec rake qc:work
 ```
-To work jobs from a custom queue:
+
+Setup a worker to work a non-default queue.
 
 ```bash
 $ QUEUE="p_queue" bundle exec rake qc:work
 ```
 
-#### Bin File
+#### Custom Worker
 
-Start by making a bin directory in your project's root directory.
-Then add an executable file called worker.
-
-**bin/worker**
+This example is probably not production ready; however, it serves as an example of how to leverage the code in the Worker class to fit your non-default requirements.
 
 ```ruby
-#!/usr/bin/env ruby
-# encoding: utf-8
-
-trap('INT')  {exit}
-trap('TERM') {exit}
-
-require "your_app"
-require "queue_classic"
-QC::Worker.new.start
-```
-
-#### Subclass QC::Worker
+require 'timeout'
+require 'queue_classic'
 
-Now that we have seen how to run a worker process, let's take a look at how to customize a worker.
-The class `QC::Worker` will probably suit most of your needs; however, there are some mechanisms
-that you will want to override. For instance, if you are using a forking worker, you will need to
-open a new database connection in the child process that is doing your work. Also, you may want to
-define how a failed job should behave. The default failed handler will simply print the job to stdout.
-You can define a failure method that will enqueue the job again, or move it to another table, etc....
+trap('INT') {exit}
+trap('TERM') {worker.stop}
 
-```ruby
-require "queue_classic"
+FailedQueue = QC::Queue.new("failed_jobs")
 
 class MyWorker < QC::Worker
-
-  # retry the job
-  def handle_failure(job, exception)
-    @queue.enqueue(job[:method], *job[:args])
-  end
-
-  # the forked proc needs a new db connection
-  def setup_child
-    ActiveRecord::Base.establish_connection
-  end
-
+ 	def handle_failure(job, e)
+		FailedQueue.enqueue(job)
+ 	end
 end
-```
 
-Notice that we have access to the `@queue` instance variable. Read the tests
-and the worker class for more information on what you can do inside of the worker.
-
-**bin/worker**
-
-```ruby
-#!/usr/bin/env ruby
-# encoding: utf-8
-
-trap('INT')  {exit}
-trap('TERM') {exit}
-
-require "your_app"
-require "queue_classic"
-require "my_worker"
-
-MyWorker.new.start
+worker = MyWorker.new(max_attempts: 10, listening_worker: true)
+
+loop do
+	job = worker.lock_job
+	Thread.new do
+		begin
+			Timeout::timeout(5) {worker.call(job)}
+		rescue => e
+			handle_failure(job, e)
+		ensure
+			worker.delete(job[:id])
+		end
+	end
+end
 ```
 
-#### QC::Worker Details
-
-##### General Idea
-
-The worker class (QC::Worker) is designed to be extended via inheritance. Any of
-its methods should be considered for extension. There are a few in particular
-that act as stubs in hopes that the user will override them. Such methods
-include: `handle_failure() and setup_child()`. See the section near the bottom
-for a detailed descriptor of how to subclass the worker.
-
-##### Algorithm
-
-When we ask the worker to start, it will enter a loop with a stop condition
-dependent upon a method named `running?` . While in the method, the worker will
-attempt to select and lock a job. If it can not on its first attempt, it will
-use an exponential back-off technique to try again.
-
-##### Signals
-
-*INT, TERM* Both of these signals will ensure that the running? method returns
-false. If the worker is waiting -- as it does per the exponential backoff
-technique; then a second signal must be sent.
-
-##### Forking
+## Setup
 
-There are many reasons why you would and would not want your worker to fork.
-An argument against forking may be that you want low latency in your job
-execution. An argument in favor of forking is that your jobs leak memory and do
-all sorts of crazy things, thus warranting the cleanup that fork allows.
-Nevertheless, forking is not enabled by default. To instruct your worker to
-fork, ensure the following shell variable is set:
+In addition to installing the rubygem, you will need to prepare your database. Database preparation includes creating a table and loading PL/pgSQL functions. You can issue the database preparation commands using `PSQL(1)` or use a database migration script.
 
-```bash
-$ export QC_FORK_WORKER='true'
-```
-
-One last note on forking. It is often the case that after Ruby forks a process,
-some sort of setup needs to be done. For instance, you may want to re-establish
-a database connection, or get a new file descriptor. queue_classic's worker
-provides a hook that is called immediately after `Kernel.fork`. To use this hook
-subclass the worker and override `setup_child()`.
-
-##### LISTEN/NOTIFY
-
-The exponential back-off algorithm will require our worker to wait if it does
-not succeed in locking a job. How we wait is something that can vary. PostgreSQL
-has a wonderful feature that we can use to wait intelligently. Processes can LISTEN on a channel and be
-alerted to notifications. queue_classic uses this feature to block until a
-notification is received. If this feature is disabled, the worker will call
-`Kernel.sleep(t)` where t is set by our exponential back-off algorithm. However,
-if we are using LISTEN/NOTIFY then we can enter a type of sleep that can be
-interrupted by a NOTIFY. For example, say we just started to wait for 2 seconds.
-After the first millisecond of waiting, a job was enqueued. With LISTEN/NOTIFY
-enabled, our worker would immediately preempt the wait and attempt to lock the job. This
-allows our worker to be much more responsive. In the case there is no
-notification, the worker will quit waiting after the timeout has expired.
-
-LISTEN/NOTIFY is disabled by default but can be enabled by setting the following shell variable:
+### Quick Start
 
 ```bash
-$ export QC_LISTENING_WORKER='true'
+$ gem install queue_classic
+$ createdb queue_classic_test
+$ export QC_DATABASE_URL="postgres://username:password@localhost/queue_classic_test"
+$ ruby -r queue_classic -e "QC::Setup.create"
+$ ruby -r queue_classic -e "QC.enqueue('Kernel.puts', 'hello world')"
+$ ruby -r queue_classic -e "QC::Worker.new.work"
 ```
 
-##### Failure
-
-I bet your worker will encounter a job that raises an exception. queue_classic
-thinks that you should know about this exception by means of you established
-exception tracker. (i.e. Hoptoad, Exceptional) To that end, queue_classic offers
-a method that you can override. This method will be passed 2 arguments: the
-exception instance and the job. Here are a few examples of things you might want
-to do inside `handle_failure()`.
-
-## Instrumentation
+### Ruby on Rails Setup
 
-QC will log elapsed time, errors and general usage in the form of data.
-To customize the output of the log data, override `QC.log(data)` and `QC.log_yield(data)`.
-By default, QC uses a simple wrapper around $stdout to put the log data in k=v
-format. For instance:
+Declare dependencies in Gemfile.
 
+```ruby
+source :rubygems
+gem "queue_classic", "2.1.0"
 ```
-lib=queue_classic level=info action=insert_job elapsed=16
-```
-
-## Tips and Tricks
-
-### Running Synchronously for tests
-
-Author: [@em_csquared](https://twitter.com/#!/em_csquared)
 
-I was testing some code that started out handling some work in a web request and
-wanted to move that work over to a queue.  After completing a red-green-refactor
-I did not want my tests to have to worry about workers or even hit the database.
-
-Turns out its easy to get queue_classic to just work in a synchronous way with:
+Require these files in your Rakefile so that you can run `rake qc:work`.
 
 ```ruby
-def QC.enqueue(function_call, *args)
-  eval("#{function_call} *#{args.inspect}")
-end
+require "queue_classic"
+require "queue_classic/tasks"
 ```
 
-Now you can test queue_classic as if it was calling your method directly!
-
-
-### Dispatching new jobs to workers without new code
-
-Author: [@ryandotsmith (ace hacker)](https://twitter.com/#!/ryandotsmith)
-
-The other day I found myself in a position in which I needed to delete a few
-thousand records. The tough part of this situation is that I needed to ensure
-the ActiveRecord callbacks were made on these objects thus making a simple SQL
-statement unfeasible. Also, I didn't want to wait all day to select and destroy
-these objects. queue_classic to the rescue! (no pun intended)
-
-The API of queue_classic enables you to quickly dispatch jobs to workers. In my
-case I wanted to call `Invoice.destroy(id)` a few thousand times. I fired up a
-Heroku console session and executed this line:
+By default, queue_classic will use the QC_DATABASE_URL falling back on DATABASE_URL. The URL must be in the following format: `postgres://username:password@localhost/database_name`.  If you use Heroku's PostgreSQL service, this will already be set. If you don't want to set this variable, you can set the connection in an initializer. **QueueClassic will maintain its own connection to the database.** This may double the number of connections to your database. Set QC::Conn.connection to share the connection between Rails & QueueClassic
 
 ```ruby
-Invoice.find(:all, :select => "id", :conditions => "some condition").map {|i| QC.enqueue("Invoice.destroy", i.id) }
+require 'queue_classic'
+QC::Conn.connection = ActiveRecord::Base.connection.raw_connection
 ```
 
-With the help of 20 workers I was able to destroy all of these records
-(preserving their callbacks) in a few minutes.
-
-### Enqueueing batches of jobs
-
-Author: [@ryandotsmith (ace hacker)](https://twitter.com/#!/ryandotsmith)
-
-I have seen several cases where the application will enqueue jobs in batches. For instance, you may be sending
-1,000 emails out. In this case, it would be foolish to do 1,000 individual transaction. Instead, you want to open
-a new transaction, enqueue all of your jobs and then commit the transaction. This will save tons of time in the
-database.
+**Note on using ActiveRecord migrations:** If you use the migration, and you wish to use commands that reset the database from the stored schema (e.g. `rake db:reset`), your application must be configured with `config.active_record.schema_format = :sql` in `config/application.rb`.  If you don't do this, the PL/pgSQL function that queue_classic creates will be lost when you reset the database.
 
-To achieve this we will create a helper method:
+```ruby
+require 'queue_classic'
 
-Now in your application code you can do something like:
+class AddQueueClassic < ActiveRecord::Migration
+  def self.up
+    QC::Setup.create
+  end
 
-```ruby
-QC::Conn.transaction do
-  Account.all.each do |act|
-    QC.enqueue("Emailer.send_notice", act.id)
+  def self.down
+    QC::Setup.drop
   end
 end
 ```
 
-### Scheduling Jobs
-
-Author: [@ryandotsmith (ace hacker)](https://twitter.com/#!/ryandotsmith)
+### Rake Task Setup
 
-Many popular queueing solution provide support for scheduling. Features like
-Redis-Scheduler and the run_at column in DJ are very important to the web
-application developer. While queue_classic does not offer any sort of scheduling
-features, I do not discount the importance of the concept. However, it is my
-belief that a scheduler has no place in a queueing library, to that end I will
-show you how to schedule jobs using queue_classic and the clockwork gem.
-
-#### Example
-
-In this example, we are working with a system that needs to compute a sales
-summary at the end of each day. Lets say that we need to compute a summary for
-each sales employee in the system.
-
-Instead of enqueueing jobs with run_at set to 24hour intervals,
-we will define a clock process to enqueue the jobs at a specified
-time on each day. Let us create a file and call it clock.rb:
-
-```ruby
-handler {|job| QC.enqueue(job)}
-every(1.day, "SalesSummaryGenerator.build_daily_report", :at => "01:00")
-```
-
-To start our scheduler, we will use the clockwork bin:
+Alternatively, you can use the Rake task to prepare your database.
 
 ```bash
-$ clockwork clock.rb
-```
-
-Now each day at 01:00 we will be sending the build_daily_report message to our
-SalesSummaryGenerator class.
-
-I found this abstraction quite powerful and easy to understand. Like
-queue_classic, the clockwork gem is simple to understand and has 0 dependencies.
-In production, I create a Heroku process type called clock. This is typically
-what my Procfile looks like:
-
-```
-worker: rake jobs:work
-clock: clockwork clock.rb
-```
-
-## Upgrading From Older Versions
-
-### 1.X to 2.X
-
-#### Database Schema Changes
-
-* all queues are in 1 table with a q_name column
-* table includes a method column and an args column
-
-#### Producer Changes
-
-* initializing a Queue instance takes a column name instead of a table name
-
-#### Consumer Changes
-
-* all of the worker configuratoin is passed in through the initializer
-* rake task uses data from env vars to initialize a worker
-
-### 0.2.X to 0.3.X
-
-* Deprecated QC.queue_length in favor of QC.length
-* Locking functions need to be loaded into database via `$ rake qc:load_functions`
-
-Also, the default queue is no longer named jobs,
-it is named queue_classic_jobs. Renaming the table is the only change that needs to be made.
+# Creating the table and functions
+$ bundle exec rake qc:create
 
-```bash
-$ psql your_database -c "ALTER TABLE jobs RENAME TO queue_classic_jobs;"
+# Dropping the table and functions
+$ bundle exec rake qc:drop
 ```
 
-Or if you are using Rails' Migrations:
-
-```ruby
-class RenameJobsTable < ActiveRecord::Migration
-
-  def self.up
-    rename_table :jobs, :queue_classic_jobs
-    remove_index :jobs, :id
-    add_index :queue_classic_jobs, :id
-  end
-
-  def self.down
-    rename_table :queue_classic_jobs, :jobs
-    remove_index :queue_classic_jobs, :id
-    add_index :jobs, :id
-  end
+## Configuration
 
-end
-```
+All configuration takes place in the form of environment vars. See [queue_classic.rb](https://github.com/ryandotsmith/queue_classic/blob/master/lib/queue_classic.rb#L23-62) for a list of options.
 
 ## Hacking on queue_classic
 
@@ -579,6 +209,12 @@ $ export QC_DATABASE_URL="postgres://username:pass@localhost/queue_classic_test"
 $ rake
 ```
 
-### License
+## License
+
+Copyright (C) 2010 Ryan Smith
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
 
-queue_classic is released under the MIT License (http://www.opensource.org/licenses/mit-license.php).
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/test/worker_test.rb

@@ -28,7 +28,7 @@ class WorkerTest < QCTest
 
   def test_work
     QC.enqueue("TestObject.no_args")
-    worker = TestWorker.new("default", 1, false, false, 1)
+    worker = TestWorker.new
     assert_equal(1, QC.count)
     worker.work
     assert_equal(0, QC.count)
@@ -37,14 +37,14 @@ class WorkerTest < QCTest
 
   def test_failed_job
     QC.enqueue("TestObject.not_a_method")
-    worker = TestWorker.new("default", 1, false, false, 1)
+    worker = TestWorker.new
     worker.work
     assert_equal(1, worker.failed_count)
   end
 
   def test_work_with_no_args
     QC.enqueue("TestObject.no_args")
-    worker = TestWorker.new("default", 1, false, false, 1)
+    worker = TestWorker.new
     r = worker.work
     assert_nil(r)
     assert_equal(0, worker.failed_count)
@@ -52,7 +52,7 @@ class WorkerTest < QCTest
 
   def test_work_with_one_arg
     QC.enqueue("TestObject.one_arg", "1")
-    worker = TestWorker.new("default", 1, false, false, 1)
+    worker = TestWorker.new
     r = worker.work
     assert_equal("1", r)
     assert_equal(0, worker.failed_count)
@@ -60,7 +60,7 @@ class WorkerTest < QCTest
 
   def test_work_with_two_args
     QC.enqueue("TestObject.two_args", "1", 2)
-    worker = TestWorker.new("default", 1, false, false, 1)
+    worker = TestWorker.new
     r = worker.work
     assert_equal(["1", 2], r)
     assert_equal(0, worker.failed_count)
@@ -69,7 +69,7 @@ class WorkerTest < QCTest
   def test_work_custom_queue
     p_queue = QC::Queue.new("priority_queue")
     p_queue.enqueue("TestObject.two_args", "1", 2)
-    worker = TestWorker.new("priority_queue", 1, false, false, 1)
+    worker = TestWorker.new(q_name: "priority_queue")
     r = worker.work
     assert_equal(["1", 2], r)
     assert_equal(0, worker.failed_count)
@@ -78,7 +78,7 @@ class WorkerTest < QCTest
   def test_worker_listens_on_chan
     p_queue = QC::Queue.new("priority_queue")
     p_queue.enqueue("TestObject.two_args", "1", 2)
-    worker = TestWorker.new("priority_queue", 1, false, true, 1)
+    worker = TestWorker.new(q_name: "priority_queue", listening_worker: true)
     r = worker.work
     assert_equal(["1", 2], r)
     assert_equal(0, worker.failed_count)
@@ -86,7 +86,7 @@ class WorkerTest < QCTest
 
   def test_worker_ueses_one_conn
     QC.enqueue("TestObject.no_args")
-    worker = TestWorker.new("default", 1, false, false, 1)
+    worker = TestWorker.new
     worker.work
     assert_equal(
       1,

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: queue_classic
 version: !ruby/object:Gem::Version
-  version: 2.0.5
+  version: 2.1.0
   prerelease: 
 platform: ruby
 authors: