queue_classic 1.0.2 → 2.0.0rc1

data/test/helper.rb

@@ -1,26 +1,30 @@
 $: << File.expand_path("lib")
 $: << File.expand_path("test")
 
-ENV['DATABASE_URL'] ||= 'postgres:///queue_classic_test'
+ENV["DATABASE_URL"] ||= "postgres:///queue_classic_test"
 
-require 'queue_classic'
-require 'database_helpers'
-
-require 'minitest/unit'
+require "queue_classic"
+require "minitest/unit"
 MiniTest::Unit.autorun
 
-def context(*args, &block)
-  return super unless (name = args.first) && block
-  klass = Class.new(MiniTest::Unit::TestCase) do
-    def self.test(name, &block)
-      define_method("test_#{name.gsub(/\W/,'_')}", &block) if block
-    end
-    def self.xtest(*args) end
-    def self.setup(&block) define_method(:setup, &block) end
-    def self.teardown(&block) define_method(:teardown, &block) end
+QC::Log.level = Logger::ERROR
+
+class QCTest < MiniTest::Unit::TestCase
+
+  def setup
+    init_db
+  end
+
+  def teardown
+    QC.delete_all
+  end
+
+  def init_db(table_name="queue_classic_jobs")
+    QC::Conn.execute("SET client_min_messages TO 'warning'")
+    QC::Conn.execute("DROP TABLE IF EXISTS #{table_name} CASCADE")
+    QC::Conn.execute("CREATE TABLE #{table_name} (id serial, q_name varchar(255), method varchar(255), args text, locked_at timestamptz)")
+    QC::Queries.load_functions
+    QC::Conn.disconnect
   end
-  (class << klass; self end).send(:define_method, :name) { name.gsub(/\W/,'_') }
 
-  klass.send :include, DatabaseHelpers
-  klass.class_eval &block
 end

data/test/queue_test.rb

@@ -1,78 +1,47 @@
 require File.expand_path("../helper.rb", __FILE__)
-require 'ostruct'
 
-context "Queue" do
+class QueueTest < QCTest
 
-  setup { @database = init_db }
-
-  test "Queue class responds to enqueue" do
-    QC::Queue.enqueue("Klass.method")
-  end
-
-  test "Queue class has a default table name" do
-    default_table_name = QC::Database.new.table_name
-    assert_equal default_table_name, QC::Queue.database.table_name
-  end
-
-  test "Queue class responds to dequeue" do
-    QC::Queue.enqueue("Klass.method")
-    assert_equal "Klass.method", QC::Queue.dequeue.signature
+  def test_enqueue
+    QC.enqueue("Klass.method")
   end
 
-  test "Queue class responds to delete" do
-    QC::Queue.enqueue("Klass.method")
-    job = QC::Queue.dequeue
-    QC::Queue.delete(job)
+  def test_lock
+    QC.enqueue("Klass.method")
+    expected = {:id=>"1", :method=>"Klass.method", :args=>[]}
+    assert_equal(expected, QC.lock)
   end
 
-  test "Queue class responds to delete_all" do
-    2.times { QC::Queue.enqueue("Klass.method") }
-    job1,job2 = QC::Queue.dequeue, QC::Queue.dequeue
-    QC::Queue.delete_all
+  def test_lock_when_empty
+    assert_nil(QC.lock)
   end
 
-  test "Queue class return the length of the queue" do
-    2.times { QC::Queue.enqueue("Klass.method") }
-    assert_equal 2, QC::Queue.length
+  def test_count
+    QC.enqueue("Klass.method")
+    assert_equal(1, QC.count)
   end
 
-  test "Queue class finds jobs using query method" do
-    QC::Queue.enqueue("Something.hard_to_find")
-    jobs = QC::Queue.query("Something.hard_to_find")
-    assert_equal 1, jobs.length
-    assert_equal "Something.hard_to_find", jobs.first.signature
+  def test_delete
+    QC.enqueue("Klass.method")
+    assert_equal(1, QC.count)
+    QC.delete(QC.lock[:id])
+    assert_equal(0, QC.count)
   end
 
-  test "queue instance responds to enqueue" do
-    QC::Queue.enqueue("Something.hard_to_find")
-    tmp_db = init_db(:custom_queue_name)
-    @queue = QC::Queue.new(:custom_queue_name)
-    @queue.enqueue "Klass.method"
-    @queue.database.disconnect
+  def test_delete_all
+    QC.enqueue("Klass.method")
+    QC.enqueue("Klass.method")
+    assert_equal(2, QC.count)
+    QC.delete_all
+    assert_equal(0, QC.count)
   end
 
-  test "queue only uses 1 connection per class" do
-    QC::Queue.length
-    QC::Queue.enqueue "Klass.method"
-    QC::Queue.delete QC::Queue.dequeue
-    QC::Queue.enqueue "Klass.method"
-    QC::Queue.dequeue
-    assert_equal 1, @database.execute("SELECT count(*) from pg_stat_activity")[0]["count"].to_i
+  def test_queue_instance
+    queue = QC::Queue.new("queue_classic_jobs", false)
+    queue.enqueue("Klass.method")
+    assert_equal(1, queue.count)
+    queue.delete(queue.lock[:id])
+    assert_equal(0, queue.count)
   end
 
-  test "Queue class enqueues a job" do
-    job = OpenStruct.new :signature => 'Klass.method', :params => ['param']
-    QC::Queue.enqueue(job)
-    dequeued_job = QC::Queue.dequeue
-    assert_equal "Klass.method", dequeued_job.signature
-    assert_equal 'param', dequeued_job.params
-  end
-
-  test "Queues have their own array" do
-    refute_equal(Class.new(QC::Queue).array, Class.new(QC::Queue).array)
-  end
-
-  test "Queues have their own database" do
-    refute_equal(Class.new(QC::Queue).database, Class.new(QC::Queue).database)
-  end
 end

data/test/worker_test.rb

@@ -1,8 +1,10 @@
 require File.expand_path("../helper.rb", __FILE__)
 
-class TestNotifier
-  def self.deliver(args={})
-  end
+module TestObject
+  extend self
+  def no_args; return nil; end
+  def one_arg(a); return a; end
+  def two_args(a,b); return [a,b]; end
 end
 
 # This not only allows me to test what happens
@@ -11,47 +13,77 @@ end
 # you want.
 class TestWorker < QC::Worker
   attr_accessor :failed_count
-  def initialize
+
+  def initialize(*args)
+    super(*args)
     @failed_count = 0
-    super
   end
+
   def handle_failure(job,e)
     @failed_count += 1
   end
 end
 
-context "Worker" do
+class WorkerTest < QCTest
 
-  setup do
-    @database = init_db
-    @worker = TestWorker.new
+  def test_work
+    QC.enqueue("TestObject.no_args")
+    worker = TestWorker.new("default", 1, false, false, 1)
+    assert_equal(1, QC.count)
+    worker.work
+    assert_equal(0, QC.count)
+    assert_equal(0, worker.failed_count)
   end
 
-  teardown do
-    @database.disconnect
+  def test_failed_job
+    QC.enqueue("TestObject.not_a_method")
+    worker = TestWorker.new("default", 1, false, false, 1)
+    worker.work
+    assert_equal(1, worker.failed_count)
   end
 
-  test "working a job" do
-    QC::Queue.enqueue "TestNotifier.deliver", {}
+  def test_work_with_no_args
+    QC.enqueue("TestObject.no_args")
+    worker = TestWorker.new("default", 1, false, false, 1)
+    r = worker.work
+    assert_nil(r)
+    assert_equal(0, worker.failed_count)
+  end
 
-    assert_equal(1, QC::Queue.length)
-    @worker.work
-    assert_equal(0, QC::Queue.length)
-    assert_equal(0, @worker.failed_count)
+  def test_work_with_one_arg
+    QC.enqueue("TestObject.one_arg", "1")
+    worker = TestWorker.new("default", 1, false, false, 1)
+    r = worker.work
+    assert_equal("1", r)
+    assert_equal(0, worker.failed_count)
   end
 
-  test "resuce failed job" do
-    QC::Queue.enqueue "TestNotifier.no_method", {}
+  def test_work_with_two_args
+    QC.enqueue("TestObject.two_args", "1", 2)
+    worker = TestWorker.new("default", 1, false, false, 1)
+    r = worker.work
+    assert_equal(["1", 2], r)
+    assert_equal(0, worker.failed_count)
+  end
 
-    @worker.work
-    assert_equal 1, @worker.failed_count
+  def test_work_custom_queue
+    p_queue = QC::Queue.new("priority_queue")
+    p_queue.enqueue("TestObject.two_args", "1", 2)
+    worker = TestWorker.new("priority_queue", 1, false, false, 1)
+    r = worker.work
+    assert_equal(["1", 2], r)
+    assert_equal(0, worker.failed_count)
   end
 
-  test "only makes one connection" do
-    QC.enqueue "TestNotifier.deliver", {}
-    @worker.work
-    assert_equal 1, @database.execute("SELECT count(*) from pg_stat_activity")[0]["count"].to_i,
+  def test_worker_ueses_one_conn
+    QC.enqueue("TestObject.no_args")
+    worker = TestWorker.new("default", 1, false, false, 1)
+    worker.work
+    assert_equal(
+      1,
+      QC::Conn.execute("SELECT count(*) from pg_stat_activity")["count"].to_i,
       "Multiple connections -- Are there other connections in other terminals?"
+    )
   end
 
 end

metadata

@@ -1,27 +1,27 @@
 --- !ruby/object:Gem::Specification
 name: queue_classic
 version: !ruby/object:Gem::Version
-  version: 1.0.2
-  prerelease: 
+  version: 2.0.0rc1
+  prerelease: 5
 platform: ruby
 authors:
 - Ryan Smith
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-08-22 00:00:00.000000000Z
+date: 2012-02-29 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
-  requirement: &15604460 !ruby/object:Gem::Requirement
+  requirement: &18736220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.11.0
+        version: 0.13.2
   type: :runtime
   prerelease: false
-  version_requirements: *15604460
+  version_requirements: *18736220
 description: queue_classic is a queueing library for Ruby apps. (Rails, Sinatra, Etc...)
   queue_classic features asynchronous job polling, database maintained locks and no
   ridiculous dependencies. As a matter of fact, queue_classic only requires pg.
@@ -31,21 +31,15 @@ extensions: []
 extra_rdoc_files: []
 files:
 - readme.md
+- lib/queue_classic/conn.rb
 - lib/queue_classic/okjson.rb
-- lib/queue_classic/logger.rb
 - lib/queue_classic/worker.rb
-- lib/queue_classic/database.rb
-- lib/queue_classic/job.rb
 - lib/queue_classic/queue.rb
 - lib/queue_classic/tasks.rb
-- lib/queue_classic/durable_array.rb
+- lib/queue_classic/queries.rb
 - lib/queue_classic.rb
-- test/database_test.rb
-- test/job_test.rb
-- test/database_helpers.rb
 - test/worker_test.rb
 - test/queue_test.rb
-- test/durable_array_test.rb
 - test/helper.rb
 homepage: http://github.com/ryandotsmith/queue_classic
 licenses: []
@@ -62,9 +56,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>='
+  - - ! '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.10
@@ -72,8 +66,5 @@ signing_key:
 specification_version: 3
 summary: postgres backed queue
 test_files:
-- test/database_test.rb
-- test/job_test.rb
 - test/worker_test.rb
 - test/queue_test.rb
-- test/durable_array_test.rb

data/lib/queue_classic/database.rb

@@ -1,188 +0,0 @@
-module QC
-  class Database
-
-    @@connection = nil
-
-    attr_reader :table_name
-    attr_reader :top_boundary
-
-    def initialize(queue_name=nil)
-      log("initialized")
-
-      @top_boundary = (ENV["QC_TOP_BOUND"] || 9).to_i
-      log("top_boundary=#{@top_boundary}")
-
-      @table_name = queue_name || "queue_classic_jobs"
-      log("table_name=#{@table_name}")
-
-      @channel_name = @table_name
-      log("channel_name=#{@channel_name}")
-
-      db_url = (ENV["QC_DATABASE_URL"] || ENV["DATABASE_URL"])
-      @db_params = URI.parse(db_url)
-      log("uri=#{db_url}")
-    end
-
-    def set_application_name
-      execute("SET application_name = 'queue_classic'")
-    end
-
-    def escape(string)
-      connection.escape(string)
-    end
-
-    def notify
-      log("NOTIFY")
-      execute("NOTIFY #{@channel_name}")
-    end
-
-    def listen
-      log("LISTEN")
-      execute("LISTEN #{@channel_name}")
-    end
-
-    def unlisten
-      log("UNLISTEN")
-      execute("UNLISTEN #{@channel_name}")
-    end
-
-    def drain_notify
-      until connection.notifies.nil?
-        log("draining notifications")
-      end
-    end
-
-    def wait_for_notify(t)
-      log("waiting for notify timeout=#{t}")
-      connection.wait_for_notify(t) {|event, pid, msg| log("received notification #{event}")}
-      log("done waiting for notify")
-    end
-
-    def transaction
-      begin
-        execute 'BEGIN'
-        yield
-        execute 'COMMIT'
-      rescue Exception
-        execute 'ROLLBACK'
-        raise
-      end
-    end
-
-    def transaction_idle?
-      connection.transaction_status == PGconn::PQTRANS_IDLE
-    end
-
-    def execute(sql, *params)
-      log("executing #{sql.inspect}, #{params.inspect}")
-      begin
-        params = nil if params.empty?
-        connection.exec(sql, params)
-      rescue PGError => e
-        log("execute exception=#{e.inspect}")
-        raise
-      end
-    end
-
-    def connection
-      @@connection ||= connect
-    end
-
-    def disconnect
-      connection.finish
-      @@connection = nil
-    end
-
-    def connect
-      log("establishing connection")
-      conn = PGconn.connect(
-        @db_params.host,
-        @db_params.port || 5432,
-        nil, '', #opts, tty
-        @db_params.path.gsub("/",""), # database name
-        @db_params.user,
-        @db_params.password
-      )
-      if conn.status != PGconn::CONNECTION_OK
-        log("connection error=#{conn.error}")
-      end
-      conn
-    end
-
-    def load_functions
-      execute(<<-EOD)
-        -- We are declaring the return type to be queue_classic_jobs.
-        -- This is ok since I am assuming that all of the users added queues will
-        -- have identical columns to queue_classic_jobs.
-        -- When QC supports queues with columns other than the default, we will have to change this.
-
-        CREATE OR REPLACE FUNCTION lock_head(tname name, top_boundary integer) RETURNS SETOF queue_classic_jobs AS $$
-        DECLARE
-          unlocked integer;
-          relative_top integer;
-          job_count integer;
-        BEGIN
-          -- The purpose is to release contention for the first spot in the table.
-          -- The select count(*) is going to slow down dequeue performance but allow
-          -- for more workers. Would love to see some optimization here...
-
-          EXECUTE 'SELECT count(*) FROM ' ||
-            '(SELECT * FROM ' || quote_ident(tname) ||
-            ' LIMIT ' || quote_literal(top_boundary) || ') limited'
-            INTO job_count;
-
-          SELECT TRUNC(random() * top_boundary + 1) INTO relative_top;
-          IF job_count < top_boundary THEN
-            relative_top = 0;
-          END IF;
-
-          LOOP
-            BEGIN
-              EXECUTE 'SELECT id FROM '
-                || quote_ident(tname)
-                || ' WHERE locked_at IS NULL'
-                || ' ORDER BY id ASC'
-                || ' LIMIT 1'
-                || ' OFFSET ' || quote_literal(relative_top)
-                || ' FOR UPDATE NOWAIT'
-              INTO unlocked;
-              EXIT;
-            EXCEPTION
-              WHEN lock_not_available THEN
-                -- do nothing. loop again and hope we get a lock
-            END;
-          END LOOP;
-
-          RETURN QUERY EXECUTE 'UPDATE '
-            || quote_ident(tname)
-            || ' SET locked_at = (CURRENT_TIMESTAMP)'
-            || ' WHERE id = $1'
-            || ' AND locked_at is NULL'
-            || ' RETURNING *'
-          USING unlocked;
-
-          RETURN;
-        END;
-        $$ LANGUAGE plpgsql;
-
-        CREATE OR REPLACE FUNCTION lock_head(tname varchar) RETURNS SETOF queue_classic_jobs AS $$
-        BEGIN
-          RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname;
-        END;
-        $$ LANGUAGE plpgsql;
-     EOD
-    end
-
-    def unload_functions
-      execute(<<-EOD)
-        DROP FUNCTION IF EXISTS lock_head(tname varchar);
-        DROP FUNCTION IF EXISTS lock_head(tname name, top_boundary integer);
-      EOD
-    end
-
-    def log(msg)
-      Logger.puts(["database", msg].join(" "))
-    end
-
-  end
-end