queue_classic 0.1.5

data/lib/queue_classic.rb

@@ -0,0 +1,15 @@
+require 'json'
+require 'pg'
+
+$: << File.expand_path("lib")
+
+require 'queue_classic/durable_array'
+require 'queue_classic/worker'
+require 'queue_classic/queue'
+require 'queue_classic/api'
+
+QC::Queue.setup :data_store => QC::DurableArray.new(:database => ENV["DATABASE_URL"])
+
+module QC
+  extend Api
+end

data/lib/queue_classic/api.rb

@@ -0,0 +1,32 @@
+module QC
+  module Api
+
+    def enqueue(job,*params)
+      Queue.enqueue(job,params)
+    end
+
+    def dequeue
+      Queue.dequeue
+    end
+
+    def delete(job)
+      Queue.delete(job)
+    end
+
+    def queue_length
+      Queue.length
+    end
+
+    def work(job)
+      klass   = job.klass
+      method  = job.method
+      params  = job.params
+
+      klass.send(method,params)
+      delete(job)
+    rescue ArgumentError => e
+      puts "ArgumentError: #{e.inspect}"
+    end
+
+  end
+end

data/lib/queue_classic/durable_array.rb

@@ -0,0 +1,124 @@
+require 'uri'
+require 'pg'
+
+module QC
+  class Job
+    attr_accessor :id, :details, :locked_at
+    def initialize(args={})
+      @id        = args["id"]
+      @details   = args["details"]
+      @locked_at = args["locked_at"]
+    end
+
+    def klass
+      Kernel.const_get(details["job"].split(".").first)
+    end
+
+    def method
+      details["job"].split(".").last
+    end
+
+    def params
+      params = details["params"]
+      if params.length == 1
+        return params[0]
+      else
+        params
+      end
+    end
+
+  end
+
+  class DurableArray
+    def initialize(args={})
+      @db_string  = args[:database]
+      @connection = connection
+      execute("SET client_min_messages TO 'warning'")
+      execute("LISTEN jobs")
+    end
+
+    def <<(details)
+      execute(
+        "INSERT INTO jobs" +
+        "(details)" +
+        "VALUES ('#{details.to_json}')"
+      )
+      execute("NOTIFY jobs, 'new-job'")
+    end
+
+    def count
+      execute("SELECT COUNT(*) from jobs")[0]["count"].to_i
+    end
+
+    def delete(job)
+      execute("DELETE FROM jobs WHERE id = #{job.id}")
+      job
+    end
+
+    def find(job)
+      find_one {"SELECT * FROM jobs WHERE id = #{job.id}"}
+    end
+
+    def head
+      find_one {"SELECT * FROM jobs ORDER BY id ASC LIMIT 1"}
+    end
+    alias :first :head
+
+    def lock_head
+      job = nil
+      @connection.transaction do
+        job = find_one {"SELECT * FROM jobs WHERE locked_at IS NULL ORDER BY id ASC LIMIT 1 FOR UPDATE"}
+        return nil unless job
+        locked  = execute("UPDATE jobs SET locked_at = (CURRENT_TIMESTAMP) WHERE id = #{job.id} AND locked_at IS NULL")
+      end
+      job
+    end
+
+    def b_head
+      if job = lock_head
+        job
+      else
+        @connection.wait_for_notify {|e,p,msg| job = lock_head if msg == "new-job" }
+        job
+      end
+    end
+
+    def each
+      execute("SELECT * FROM jobs ORDER BY id ASC").each do |r|
+        yield(JSON.parse(r["details"]))
+      end
+    end
+
+    def execute(sql)
+      @connection.async_exec(sql)
+    end
+
+    def find_one
+      res = execute(yield)
+      if res.cmd_tuples > 0
+        res.map do |r|
+          Job.new(
+            "id"        => r["id"],
+            "details"   => JSON.parse( r["details"]),
+            "locked_at" => r["locked_at"]
+          )
+        end.pop
+      end
+    end
+
+    def connection
+      db_params = URI.parse(@db_string)
+      if db_params.scheme == "postgres"
+        PGconn.connect(
+          :dbname   => db_params.path.gsub("/",""),
+          :user     => db_params.user,
+          :password => db_params.password,
+          :host     => db_params.host
+        )
+      else
+        PGconn.connect(:dbname => @db_string)
+      end
+    end
+
+  end
+end

data/lib/queue_classic/queue.rb

@@ -0,0 +1,25 @@
+module  QC
+  class Queue
+    def self.setup(args={})
+      @@data = args[:data_store] || []
+      self
+    end
+
+    def self.enqueue(job,params)
+      @@data << {"job" => job, "params" => params}
+    end
+
+    def self.dequeue
+      @@data.b_head
+    end
+
+    def self.delete(job)
+      @@data.delete(job)
+    end
+
+    def self.length
+      @@data.count
+    end
+
+  end
+end

data/lib/queue_classic/tasks.rb

@@ -0,0 +1,13 @@
+namespace :jobs do
+  task :work  => :environment do
+    QC::Worker.new.start
+  end
+end
+namespace :qc do
+  task :work do
+    QC::Worker.new.start
+  end
+  task :jobs do
+    QC.queue_length
+  end
+end

data/lib/queue_classic/worker.rb

@@ -0,0 +1,25 @@
+module QC
+  class Worker
+
+    def initialize
+      @worker_id = rand(1000)
+    end
+
+    def start
+      puts "#{@worker_id} ready for work"
+      loop { work }
+    end
+
+    def work
+      job = QC.dequeue
+      # if we are here, dequeue has unblocked
+      # and we may have a job.
+      if job
+        puts "#{@worker_id} working job"
+        QC.work(job)
+      end
+
+    end
+
+  end
+end

data/readme.markdown

@@ -0,0 +1,118 @@
+# Queue Classic
+__Alpha 0.1.5__
+
+_Queue Classic 0.1.5 is not ready for production. However, it is under active development and I expect a beta release within the following months._
+
+Queue Classic is an alternative queueing library for Ruby apps (Rails, Sinatra, Etc...) Queue Classic features __asynchronous__ job polling, database maintained locks and
+no ridiculous dependencies. As a matter of fact, Queue Classic only requires the __pg__ and __json__.
+
+## Installation
+
+### TL;DR
+1. gem install queue_classic
+2. add jobs table to your database
+3. QC.enqueue "Class.method", :arg1 => val1
+4. rake qc:work
+
+### Gem
+
+    gem install queue_classic
+
+### Database
+
+Queue Classic needs a database, so make sure that DATABASE_URL points to your database. If you are unsure about whether this var is set, run:
+    echo $DATABASE_URL
+in your shell. If you are using Heroku, this var is set and pointing to your primary database.
+
+Once your Database is set, you will need to add a jobs table. If you are using rails, add a migration with the following tables:
+
+    class CreateJobsTable < ActiveRecord::Migration
+      def self.up
+        create_table :jobs do |t|
+          t.text :details
+          t.timestamp :locked_at
+        end
+      end
+
+      def self.down
+        drop_table :jobs
+      end
+    end
+After running this migration, your database should be ready to go. As a sanity check, enqueue a job and then issue a SELECT in the postgres console.
+
+__script/console__
+    QC.enqueue "Class.method"
+__Terminal__
+    psql you_database_name
+    select * from jobs;
+You should see the job "Class.method"
+
+### Rakefile
+
+As a convenience, I added a rake task that responds to `rake jobs:work` There are also rake tasks in the `qc` name space.
+To get access to these tasks, Add `require 'queue_classic/tasks'` to your Rakefile.
+
+## Fundamentals
+
+### Enqueue
+
+To place a job onto the queue, you should specify a class and a class method. The syntax should be:
+
+    QC.enqueue('Class.method', :arg1 => 'value1', :arg2 => 'value2')
+
+The job gets stored in the jobs table with a details field set to: `{ job: Class.method, params: {arg1: value1, arg2: value2}}` (as JSON)
+Class can be any class and method can be anything that Class will respond to. For example:
+
+    class Invoice < ActiveRecord::Base
+      def self.process(invoice_id)
+        invoice = find(invoice_id)
+        invoice.process!
+      end
+
+      def self.process_all
+        Invoice.all do |invoice|
+          QC.enqueue "Invoice.process", invoice.id
+        end
+      end
+    end
+
+
+### Dequeue
+
+Traditionally, a queue's dequeue operation will remove the item from the queue. However, Queue Classic will not delete the item from the queue, it will lock it
+and then the worker will delete the job once it has finished working it. Queue Classic's greatest strength is it's ability to safely lock jobs. Unlike other
+database backed queing libraries, Classic Queue uses the database time to lock. This allows you to be more relaxed about the time synchronization of your worker machines.
+
+Finally, the strongest feature of Queue Classic is it's ability to block on on dequeue. This design removes the need to __ Sleep & SELECT. __ Queue Classic takes advantage
+of the wonderul PUB/SUB featuers built in to Postgres. Basically there is a channel in which the workers LISTEN. When a new job is added to the queue, the queue sends NOTIFY
+messages on the channel. Once a NOTIFY is sent, each worker races to acquire a lock on a job. A job is awareded to the victor while the rest go back to wait for another job.
+
+## FAQ
+
+How is this different than DJ?
+> TL;DR = Store job as JSON (better introspection), Queue manages the time for locking jobs (workers can be out of sync.), No magic (less code), Small footprint (ORM Free).
+
+> __Introspection__ I want the data in the queue to be as simple as possible. Since we only store the Class, Method and Args, introspection into the queue is
+quite simpler.
+
+> __Locking__ You might have noticed that DJ's worker calls Time.now(). In a cloud environment, this could allow for workers to be confused about
+the status of a job. Classic Queue locks a job using Postgres' TIMESTAMP function.
+
+> __Magic__ I find disdain for methods on my objects that have nothing to do with the purpose of the object. Methods like "should" and "delay"
+are quite distasteful and obscure what is actually going on. If you use TestUnit for this reason, you might like Queue Classic. Anyway, I think
+the fundamental concept of a message queue is not that difficult to grasp; therefore, I have taken the time to make Queue Classic as transparent as possilbe.
+
+> __Footprint__ You don't need ActiveRecord or any other ORM to find the head or add to the tail. Take a look at the DurableArray class to see the SQL Classic Queue employees.
+
+Why doesn't your queue retry failed jobs?
+> I believe the Class method should handle any sort of exception.  Also, I think
+that the model you are working on should know about it's state. For instance, if you are
+creating jobs for the emailing of newsletters; put a emailed_at column on your newsletter model
+and then right before the job quits, touch the emailed_at column.
+
+Can I use this library with 50 Heroku Workers?
+> Maybe. I haven't tested 50 workers yet. But it is definitely a goal for Queue Classic. I am not sure when,
+but you can count on this library being able to handle all Heroku can throw at it.
+
+Why does this project seem incomplete? Will you make it production ready?
+> I started this project on 1/24/2011. Check back soon! Also, feel free to contact me to find out how passionate I am about queueing.

data/test/api_test.rb

@@ -0,0 +1,14 @@
+require File.expand_path("../helper.rb", __FILE__)
+
+class ApiTest < MiniTest::Unit::TestCase
+  include DatabaseHelpers
+
+  def test_enqueue_takes_a_job
+    clean_database
+
+    assert_equal 0, QC.queue_length
+    res = QC.enqueue "Notifier.send", {}
+    assert_equal 1, QC.queue_length
+  end
+end
+

data/test/database_helpers.rb

@@ -0,0 +1,48 @@
+module DatabaseHelpers
+
+  def clean_database
+    drop_table
+    create_table
+    disconnect
+  end
+
+  def create_database
+    postgres.exec "CREATE DATABASE #{ENV['DATABASE_URL']}"
+  end
+
+  def drop_database
+    postgres.exec "DROP DATABASE IF EXISTS #{ENV['DATABASE_URL']}"
+  end
+
+  def create_table
+    jobs_db.exec(
+      "CREATE TABLE jobs"    +
+      "("                    +
+      "id        SERIAL,"    +
+      "details   text,"      +
+      "locked_at timestamp without time zone" +
+      ");"
+    )
+  end
+
+  def drop_table
+    jobs_db.exec("DROP TABLE IF EXISTS jobs")
+  end
+
+  def disconnect
+    jobs_db.finish
+    postgres.finish
+  end
+
+  def jobs_db
+    @jobs_db ||= PGconn.connect(:dbname => ENV["DATABASE_URL"])
+    @jobs_db.exec("SET client_min_messages TO 'warning'")
+    @jobs_db
+  end
+
+  def postgres
+    @postgres ||= PGconn.connect(:dbname => 'postgres')
+    @postgres.exec("SET client_min_messages TO 'warning'")
+    @postgres
+  end
+end

data/test/durable_array_test.rb

@@ -0,0 +1,90 @@
+require File.expand_path("../helper.rb", __FILE__)
+
+class DurableArrayTest < MiniTest::Unit::TestCase
+  include DatabaseHelpers
+
+  def test_head_decodes_json
+    clean_database
+    array = QC::DurableArray.new(:database => "queue_classic_test")
+
+    array << {"test" => "ok"}
+    assert_equal({"test" => "ok"}, array.head.details)
+  end
+
+  def test_count_returns_number_of_rows
+    clean_database
+    array = QC::DurableArray.new(:database => "queue_classic_test")
+
+    array << {"test" => "ok"}
+    assert_equal 1, array.count
+    array << {"test" => "ok"}
+    assert_equal 2, array.count
+  end
+
+  def test_head_returns_first_job
+    clean_database
+    array = QC::DurableArray.new(:database => "queue_classic_test")
+
+    job = {"job" => "one"}
+    array << job
+    assert_equal job, array.head.details
+  end
+
+  def test_head_returns_first_job_when_many_are_in_array
+    clean_database
+    array = QC::DurableArray.new(:database => "queue_classic_test")
+
+    array << {"job" => "one"}
+    array << {"job" => "two"}
+    assert_equal({"job" => "one"}, array.head.details)
+  end
+
+  def test_delete_removes_job_from_array
+    clean_database
+    array = QC::DurableArray.new(:database => "queue_classic_test")
+
+    array << {"job" => "one"}
+    assert_equal( {"job" => "one"}, array.head.details)
+    array.delete(array.head)
+    assert_nil array.head
+  end
+
+  def test_delete_returns_job_after_delete
+    clean_database
+    array = QC::DurableArray.new(:database => "queue_classic_test")
+
+    array << {"job" => "one"}
+    assert_equal({"job" => "one"}, array.head.details)
+
+    res = array.delete(array.head)
+    assert_nil(array.head)
+    assert_equal({"job" => "one"}, res.details)
+  end
+
+  def test_each_yields_the_details_for_each_job
+    clean_database
+    array = QC::DurableArray.new(:database => "queue_classic_test")
+
+    array << {"job" => "one"}
+    array << {"job" => "two"}
+    results = []
+    array.each {|v| results << v}
+    assert_equal([{"job" => "one"},{"job" => "two"}], results)
+  end
+
+  def test_connection_builds_db_connection_for_uri
+    array = QC::DurableArray.new(:database => "postgres://ryandotsmith:@localhost/queue_classic_test")
+    assert_equal "ryandotsmith", array.connection.user
+    assert_equal "localhost", array.connection.host
+    assert_equal "queue_classic_test", array.connection.db
+  end
+
+  def test_connection_builds_db_connection_for_database
+    array = QC::DurableArray.new(:database => "queue_classic_test")
+
+    # FIXME not everyone will have a postgres user named: ryandotsmith
+    assert_equal "ryandotsmith", array.connection.user
+    assert_equal "queue_classic_test", array.connection.db
+  end
+
+end

data/test/helper.rb

@@ -0,0 +1,8 @@
+$: << File.expand_path("lib")
+$: << File.expand_path("test")
+
+require 'queue_classic'
+require 'database_helpers'
+
+require 'minitest/unit'
+MiniTest::Unit.autorun

data/test/queue_test.rb

@@ -0,0 +1,5 @@
+require File.expand_path("../helper.rb", __FILE__)
+
+class QueueTest < MiniTest::Unit::TestCase
+  include DatabaseHelpers
+end

data/test/worker_test.rb

@@ -0,0 +1,22 @@
+require File.expand_path("../helper.rb", __FILE__)
+
+class TestNotifier
+  def self.deliver(args={})
+  end
+end
+
+class WorkerTest < MiniTest::Unit::TestCase
+  include DatabaseHelpers
+
+  def test_working_a_job
+    clean_database
+
+    QC.enqueue "TestNotifier.deliver", {}
+    worker = QC::Worker.new
+
+    assert_equal(1, QC.queue_length)
+    worker.work
+    assert_equal(0, QC.queue_length)
+  end
+
+end

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification 
+name: queue_classic
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 5
+  version: 0.1.5
+platform: ruby
+authors: 
+- Ryan Smith
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-01-28 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: pg
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+description: Queue Classic is an alternative queueing library for Ruby apps (Rails, Sinatra, Etc...) Queue Classic features asynchronous job polling, database maintained locks and no ridiculous dependencies. As a matter of fact, Queue Classic only requires the pg and json.
+email: ryan@heroku.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- readme.markdown
+- lib/queue_classic/api.rb
+- lib/queue_classic/durable_array.rb
+- lib/queue_classic/queue.rb
+- lib/queue_classic/tasks.rb
+- lib/queue_classic/worker.rb
+- lib/queue_classic.rb
+- test/api_test.rb
+- test/database_helpers.rb
+- test/durable_array_test.rb
+- test/helper.rb
+- test/queue_test.rb
+- test/worker_test.rb
+has_rdoc: true
+homepage: http://github.com/ryandotsmith/Queue-Classic
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Queue Classic is an alternative queueing library for Ruby apps (Rails, Sinatra, Etc...) Queue Classic features asynchronous job polling, database maintained locks and no ridiculous dependencies. As a matter of fact, Queue Classic only requires the pg and json.(simple)
+test_files: 
+- test/api_test.rb
+- test/durable_array_test.rb
+- test/queue_test.rb
+- test/worker_test.rb