queue_classic 0.1.6 → 0.2.0

data/lib/notifier.rb

@@ -1,5 +1,7 @@
 class Notifier
+
   def self.deliver(msg)
-  `say #{msg}`
+    `say #{msg}`
   end
+
 end

data/lib/queue_classic.rb

@@ -1,5 +1,6 @@
 require 'json'
 require 'pg'
+require 'uri'
 
 $: << File.expand_path("lib")
 
@@ -7,9 +8,10 @@ require 'queue_classic/durable_array'
 require 'queue_classic/worker'
 require 'queue_classic/queue'
 require 'queue_classic/api'
+require 'queue_classic/job'
 
 module QC
   extend Api
 end
 
-QC::Queue.setup :data_store => QC::DurableArray.new(:database => ENV["DATABASE_URL"])
+QC::Queue.instance.setup :data_store => QC::DurableArray.new(:database => ENV["DATABASE_URL"])

data/lib/queue_classic/api.rb

@@ -1,20 +1,24 @@
 module QC
   module Api
 
+    def queue
+      @queue ||= Queue.instance
+    end
+
     def enqueue(job,*params)
-      Queue.enqueue(job,params)
+      queue.enqueue(job,params)
     end
 
     def dequeue
-      Queue.dequeue
+      queue.dequeue
     end
 
     def delete(job)
-      Queue.delete(job)
+      queue.delete(job)
     end
 
     def queue_length
-      Queue.length
+      queue.length
     end
 
     def work(job)
@@ -23,13 +27,10 @@ module QC
       params  = job.params
 
       klass.send(method,params)
-      delete(job)
-    rescue ArgumentError => e
-      puts "ArgumentError: #{e.inspect}"
     end
 
     def logging_enabled?
-      true
+      ENV["LOGGING"]
     end
 
   end

data/lib/queue_classic/durable_array.rb

@@ -1,35 +1,6 @@
-require 'uri'
-require 'pg'
-
 module QC
-  class Job
-    attr_accessor :id, :details, :locked_at
-    def initialize(args={})
-      @id        = args["id"]
-      @details   = args["details"]
-      @locked_at = args["locked_at"]
-    end
-
-    def klass
-      Kernel.const_get(details["job"].split(".").first)
-    end
-
-    def method
-      details["job"].split(".").last
-    end
-
-    def params
-      params = details["params"]
-      if params.length == 1
-        return params[0]
-      else
-        params
-      end
-    end
-
-  end
-
   class DurableArray
+
     def initialize(args={})
       @db_string  = args[:database]
       @connection = connection
@@ -55,26 +26,17 @@ module QC
       find_one {"SELECT * FROM jobs WHERE id = #{job.id}"}
     end
 
-    def head
-      find_one {"SELECT * FROM jobs ORDER BY id ASC LIMIT 1"}
-    end
-    alias :first :head
-
     def lock_head
       job = nil
-      with_log("start lock transaction") do
-        @connection.transaction do
-          if job = find_one {"SELECT * FROM jobs WHERE locked_at IS NULL ORDER BY id ASC LIMIT 1 FOR UPDATE"}
-            with_log("lock acquired for #{job.inspect}") do
-              execute("UPDATE jobs SET locked_at = (CURRENT_TIMESTAMP) WHERE id = #{job.id} AND locked_at IS NULL")
-            end
-          end
+      @connection.transaction do
+        if job = find_one {"SELECT * FROM jobs WHERE locked_at IS NULL ORDER BY id ASC LIMIT 1 FOR UPDATE"}
+          execute("UPDATE jobs SET locked_at = (CURRENT_TIMESTAMP) WHERE id = #{job.id} AND locked_at IS NULL")
         end
       end
       job
     end
 
-    def b_head
+    def first
       if job = lock_head
         job
       else

data/lib/queue_classic/job.rb

@@ -0,0 +1,29 @@
+module QC
+  class Job
+    attr_accessor :id, :details, :locked_at
+
+    def initialize(args={})
+      @id        = args["id"]
+      @details   = args["details"]
+      @locked_at = args["locked_at"]
+    end
+
+    def klass
+      Kernel.const_get(details["job"].split(".").first)
+    end
+
+    def method
+      details["job"].split(".").last
+    end
+
+    def params
+      params = details["params"]
+      if params.length == 1
+        return params[0]
+      else
+        params
+      end
+    end
+
+  end
+end

data/lib/queue_classic/queue.rb

@@ -1,25 +1,26 @@
+require 'singleton'
+
 module  QC
   class Queue
-    def self.setup(args={})
-      @@data = args[:data_store] || []
-      self
+    include Singleton
+    def setup(args={})
+      @data = args[:data_store]
     end
 
-    def self.enqueue(job,params)
-      @@data << {"job" => job, "params" => params}
+    def enqueue(job,params)
+      @data << {"job" => job, "params" => params}
     end
 
-    def self.dequeue
-      @@data.b_head
+    def dequeue
+      @data.first
     end
 
-    def self.delete(job)
-      @@data.delete(job)
+    def delete(job)
+      @data.delete(job)
     end
 
-    def self.length
-      @@data.count
+    def length
+      @data.count
     end
-
   end
 end

data/lib/queue_classic/worker.rb

@@ -1,24 +1,29 @@
 module QC
   class Worker
 
-    def initialize
-      @worker_id = rand(1000)
-    end
-
     def start
-      puts "#{@worker_id} ready for work"
       loop { work }
     end
 
     def work
-      job = QC.dequeue
-      # if we are here, dequeue has unblocked
-      # and we may have a job.
-      if job
-        puts "#{@worker_id} working job"
-        QC.work(job)
+      if job = QC.dequeue #blocks until we have a job
+        begin
+          QC.work(job)
+        rescue Object => e
+          handle_failure(job,e)
+        ensure
+          QC.delete(job)
+        end
       end
+    end
 
+    #override this method to do whatever you want
+    def handle_failure(job,e)
+      puts "!"
+      puts "! \t FAIL"
+      puts "! \t \t #{job.inspect}"
+      puts "! \t \t #{e.inspect}"
+      puts "!"
     end
 
   end

data/readme.markdown

@@ -1,7 +1,9 @@
 # Queue Classic
-__Alpha 0.1.6__
+__Beta 0.2.0__
 
-_Queue Classic 0.1.6 is not ready for production. However, it is under active development and I expect a beta release within the following months._
+__Queue Classic 0.2.0 is in Beta.__ I have been using this library with 30-50 Heroku workers and have had great results. However, your mileage may vary.
+
+I am using this in production applications and plan to maintain and support this library for a long time.
 
 Queue Classic is an alternative queueing library for Ruby apps (Rails, Sinatra, Etc...) Queue Classic features __asynchronous__ job polling, database maintained locks and
 no ridiculous dependencies. As a matter of fact, Queue Classic only requires the __pg__ and __json__.
@@ -14,6 +16,12 @@ no ridiculous dependencies. As a matter of fact, Queue Classic only requires the
 3. QC.enqueue "Class.method", :arg1 => val1
 4. rake qc:work
 
+### Dependencies
+
+  Postgres version 9
+
+  Ruby (gems: pg, json)
+
 ### Gem
 
     gem install queue_classic
@@ -31,6 +39,7 @@ Once your Database is set, you will need to add a jobs table. If you are using r
         create_table :jobs do |t|
           t.text :details
           t.timestamp :locked_at
+          t.index :id
         end
       end
 
@@ -40,6 +49,9 @@ Once your Database is set, you will need to add a jobs table. If you are using r
     end
 After running this migration, your database should be ready to go. As a sanity check, enqueue a job and then issue a SELECT in the postgres console.
 
+Be sure and add the index to the id column. This will help out the worker if the queue should ever reach an obscene length. It made a huge difference
+when running the benchmark.
+
 __script/console__
     QC.enqueue "Class.method"
 __Terminal__
@@ -87,13 +99,40 @@ Finally, the strongest feature of Queue Classic is it's ability to block on on d
 of the wonderul PUB/SUB featuers built in to Postgres. Basically there is a channel in which the workers LISTEN. When a new job is added to the queue, the queue sends NOTIFY
 messages on the channel. Once a NOTIFY is sent, each worker races to acquire a lock on a job. A job is awareded to the victor while the rest go back to wait for another job.
 
+### The Worker
+
+The worker calls dequeue and then calls the enqueued method with the supplied arguments. Once the method terminates, the job is deleted from the queue. In the case that your method
+does not terminate, or the worker unexpectingly dies, Queue Classic will do following: It will ensure that the job is deleted from the queue and it will call a method (which is certainly over
+rideable) that handles the failure. You can do whatever you want in the handle_failure method, log to Get Exceptional, enqueue the job again, log to stderror.
+
+    class MyWorker < QC::Worker
+      def handle_failure(job,exception)
+        puts job.inspect
+        puts exception.inspect
+      end
+    end
+
+    worker = MyWorker.new
+    worker.start
+
+## Performance
+I am pleased at the performance of Queue Classic. It ran 3x faster than the some of the most popular Relational Database backed queues. (I have yet to benchmark redis backed queues)
+
+    ruby benchmark.rb
+                    user     system      total        real
+                0.950000   0.620000   1.570000 (  9.479941)
+
+Hardware: Mac Book Pro 2.8 GHz Intel Core i7. SSD. 4 GB memory.
+
+Software: Ruby 1.9.2-p0, PostgreSQL 9.0.2
+
 ## FAQ
 
 How is this different than DJ?
 > TL;DR = Store job as JSON (better introspection), Queue manages the time for locking jobs (workers can be out of sync.), No magic (less code), Small footprint (ORM Free).
 
 > __Introspection__ I want the data in the queue to be as simple as possible. Since we only store the Class, Method and Args, introspection into the queue is
-quite simpler.
+quite simple.
 
 > __Locking__ You might have noticed that DJ's worker calls Time.now(). In a cloud environment, this could allow for workers to be confused about
 the status of a job. Classic Queue locks a job using Postgres' TIMESTAMP function.
@@ -111,8 +150,7 @@ creating jobs for the emailing of newsletters; put a emailed_at column on your n
 and then right before the job quits, touch the emailed_at column.
 
 Can I use this library with 50 Heroku Workers?
-> Maybe. I haven't tested 50 workers yet. But it is definitely a goal for Queue Classic. I am not sure when,
-but you can count on this library being able to handle all Heroku can throw at it.
+> Yes.
 
 Why does this project seem incomplete? Will you make it production ready?
 > I started this project on 1/24/2011. Check back soon! Also, feel free to contact me to find out how passionate I am about queueing.

data/test/database_helpers.rb

@@ -23,6 +23,7 @@ module DatabaseHelpers
       "locked_at timestamp without time zone" +
       ");"
     )
+    jobs_db.exec("CREATE INDEX jobs_id_idx ON jobs (id)")
   end
 
   def drop_table

data/test/durable_array_test.rb

@@ -3,12 +3,12 @@ require File.expand_path("../helper.rb", __FILE__)
 class DurableArrayTest < MiniTest::Unit::TestCase
   include DatabaseHelpers
 
-  def test_head_decodes_json
+  def test_first_decodes_json
     clean_database
     array = QC::DurableArray.new(:database => "queue_classic_test")
 
     array << {"test" => "ok"}
-    assert_equal({"test" => "ok"}, array.head.details)
+    assert_equal({"test" => "ok"}, array.first.details)
   end
 
   def test_count_returns_number_of_rows
@@ -21,22 +21,22 @@ class DurableArrayTest < MiniTest::Unit::TestCase
     assert_equal 2, array.count
   end
 
-  def test_head_returns_first_job
+  def test_first_returns_fsrst_job
     clean_database
     array = QC::DurableArray.new(:database => "queue_classic_test")
 
     job = {"job" => "one"}
     array << job
-    assert_equal job, array.head.details
+    assert_equal job, array.first.details
   end
 
-  def test_head_returns_first_job_when_many_are_in_array
+  def test_first_returns_first_job_when_many_are_in_array
     clean_database
     array = QC::DurableArray.new(:database => "queue_classic_test")
 
     array << {"job" => "one"}
     array << {"job" => "two"}
-    assert_equal({"job" => "one"}, array.head.details)
+    assert_equal({"job" => "one"}, array.first.details)
   end
 
   def test_delete_removes_job_from_array
@@ -44,9 +44,11 @@ class DurableArrayTest < MiniTest::Unit::TestCase
     array = QC::DurableArray.new(:database => "queue_classic_test")
 
     array << {"job" => "one"}
-    assert_equal( {"job" => "one"}, array.head.details)
-    array.delete(array.head)
-    assert_nil array.head
+    job = array.first
+
+    assert_equal( {"job" => "one"}, job.details)
+    array.delete(job)
+    assert_nil array.first
   end
 
   def test_delete_returns_job_after_delete
@@ -54,10 +56,12 @@ class DurableArrayTest < MiniTest::Unit::TestCase
     array = QC::DurableArray.new(:database => "queue_classic_test")
 
     array << {"job" => "one"}
-    assert_equal({"job" => "one"}, array.head.details)
+    job = array.first
+
+    assert_equal({"job" => "one"}, job.details)
 
-    res = array.delete(array.head)
-    assert_nil(array.head)
+    res = array.delete(job)
+    assert_nil(array.first)
     assert_equal({"job" => "one"}, res.details)
   end
 

data/test/helper.rb

@@ -1,8 +1,18 @@
 $: << File.expand_path("lib")
 $: << File.expand_path("test")
 
+ENV["DATABASE_URL"] = 'queue_classic_test'
+
 require 'queue_classic'
 require 'database_helpers'
 
 require 'minitest/unit'
 MiniTest::Unit.autorun
+
+def set_data_store(store=nil)
+  QC::Queue.instance.setup(
+    :data_store => (
+      store || QC::DurableArray.new(:database => ENV["DATABASE_URL"])
+    )
+  )
+end

data/test/queue_test.rb

@@ -2,4 +2,21 @@ require File.expand_path("../helper.rb", __FILE__)
 
 class QueueTest < MiniTest::Unit::TestCase
   include DatabaseHelpers
+
+  def test_queue_is_singleton
+    assert_equal QC::Queue, QC::Queue.instance.class
+  end
+
+  def test_queue_setup
+    QC::Queue.instance.setup :data_store => []
+    assert_equal [], QC::Queue.instance.instance_variable_get(:@data)
+  end
+
+  def test_queue_length
+    QC::Queue.instance.setup :data_store => []
+    QC::Queue.instance.enqueue "job","params"
+
+    assert_equal 1, QC::Queue.instance.length
+  end
+
 end

data/test/worker_test.rb

@@ -5,18 +5,46 @@ class TestNotifier
   end
 end
 
+# This not only allows me to test what happens
+# when a failure occures but it also demonstrates
+# how to override the worker to handle failures the way
+# you want.
+class TestWorker < QC::Worker
+  attr_accessor :failed_count
+  def initialize
+    @failed_count = 0
+    super
+  end
+  def handle_failure(job,e)
+    @failed_count += 1
+  end
+end
+
 class WorkerTest < MiniTest::Unit::TestCase
   include DatabaseHelpers
 
   def test_working_a_job
+    set_data_store
     clean_database
 
     QC.enqueue "TestNotifier.deliver", {}
-    worker = QC::Worker.new
+    worker = TestWorker.new
 
     assert_equal(1, QC.queue_length)
     worker.work
     assert_equal(0, QC.queue_length)
+    assert_equal(0, worker.failed_count)
+  end
+
+  def test_rescue_failed_jobs
+    set_data_store
+    clean_database
+
+    QC.enqueue "TestNotifier.no_method", {}
+    worker = TestWorker.new
+
+    worker.work
+    assert_equal 1, worker.failed_count
   end
 
 end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 1
-  - 6
-  version: 0.1.6
+  - 2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Ryan Smith
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-01-28 00:00:00 -08:00
+date: 2011-02-16 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -58,6 +58,7 @@ files:
 - lib/notifier.rb
 - lib/queue_classic/api.rb
 - lib/queue_classic/durable_array.rb
+- lib/queue_classic/job.rb
 - lib/queue_classic/queue.rb
 - lib/queue_classic/tasks.rb
 - lib/queue_classic/worker.rb