queue_classic 3.1.0 → 3.2.0.RC1

data/lib/queue_classic/config.rb

@@ -0,0 +1,85 @@
+module QC
+  module Config
+    # You can use the APP_NAME to query for
+    # postgres related process information in the
+    # pg_stat_activity table.
+    def app_name
+      @app_name ||= ENV["QC_APP_NAME"] || "queue_classic"
+    end
+
+    # Number of seconds to block on the listen chanel for new jobs.
+    def wait_time
+      @wait_time ||= (ENV["QC_LISTEN_TIME"] || 5).to_i
+    end
+
+    # Why do you want to change the table name?
+    # Just deal with the default OK?
+    # If you do want to change this, you will
+    # need to update the PL/pgSQL lock_head() function.
+    # Come on. Don't do it.... Just stick with the default.
+    def table_name
+      @table_name ||= "queue_classic_jobs"
+    end
+
+    def queue
+      @queue = ENV["QUEUE"] || "default"
+    end
+
+    # The default queue used by `QC.enqueue`.
+    def default_queue
+      @default_queue ||= Queue.new(QC.queue)
+    end
+
+    def default_queue=(queue)
+      @default_queue = queue
+    end
+
+    # Each row in the table will have a column that
+    # notes the queue. You can point your workers
+    # at different queues.
+    def queues
+      @queues ||= (ENV["QUEUES"] && ENV["QUEUES"].split(",").map(&:strip)) || []
+    end
+
+    # Set this to 1 for strict FIFO.
+    # There is nothing special about 9....
+    def top_bound
+      @top_bound ||= (ENV["QC_TOP_BOUND"] || 9).to_i
+    end
+
+    # Set this variable if you wish for
+    # the worker to fork a UNIX process for
+    # each locked job. Remember to re-establish
+    # any database connections. See the worker
+    # for more details.
+    def fork_worker?
+      @fork_worker ||= (!ENV["QC_FORK_WORKER"].nil?)
+    end
+
+    # The worker class instantiated by QC's rake tasks.
+    def default_worker_class
+
+      @worker_class ||= (ENV["QC_DEFAULT_WORKER_CLASS"] && Kernel.const_get(ENV["QC_DEFAULT_WORKER_CLASS"]) ||
+                         QC::Worker)
+
+    end
+
+    def default_worker_class=(worker_class)
+      @worker_class = worker_class
+    end
+
+    # reset memoized configuration
+    def reset_config
+      # TODO: we might want to think about storing these in a Hash.
+      @app_name = nil
+      @wait_time = nil
+      @table_name = nil
+      @queue = nil
+      @default_queue = nil
+      @queues = nil
+      @top_bound = nil
+      @fork_worker = nil
+      @worker_class = nil
+    end
+  end
+end

data/lib/queue_classic/conn_adapter.rb

@@ -48,6 +48,13 @@ module QC
       end
     end
 
+    def server_version
+      @server_version ||= begin
+                            version = execute("SHOW server_version_num;")["server_version_num"]
+                            version && version.to_i
+                          end
+    end
+
     private
 
     def wait_for_notify(t)
@@ -74,7 +81,7 @@ module QC
       if conn.status != PGconn::CONNECTION_OK
         QC.log(:error => conn.error)
       end
-      conn.exec("SET application_name = '#{QC::APP_NAME}'")
+      conn.exec("SET application_name = '#{QC.app_name}'")
       conn
     end
 

data/lib/queue_classic/queue.rb

@@ -9,7 +9,7 @@ module QC
     attr_reader :name, :top_bound
     def initialize(name, top_bound=nil)
       @name = name
-      @top_bound = top_bound || QC::TOP_BOUND
+      @top_bound = top_bound || QC.top_bound
     end
 
     def conn_adapter=(a)
@@ -34,9 +34,10 @@ module QC
     # The args are stored as a collection and then splatted inside the worker.
     # Examples of args include: `'hello world'`, `['hello world']`,
     # `'hello', 'world'`.
+    # This method returns a hash with the id of the enqueued job.
     def enqueue(method, *args)
       QC.log_yield(:measure => 'queue.enqueue') do
-        s = "INSERT INTO #{TABLE_NAME} (q_name, method, args) VALUES ($1, $2, $3)"
+        s = "INSERT INTO #{QC.table_name} (q_name, method, args) VALUES ($1, $2, $3) RETURNING id"
         conn_adapter.execute(s, name, method, JSON.dump(args))
       end
     end
@@ -46,8 +47,9 @@ module QC
     # The time argument must be a Time object or a float timestamp. The method
     # and args argument must be in the form described in the documentation for
     # the #enqueue method.
+    # This method returns a hash with the id of the enqueued job.
     def enqueue_at(timestamp, method, *args)
-      offset = Time.at(timestamp) - Time.now
+      offset = Time.at(timestamp).to_i - Time.now.to_i
       enqueue_in(offset, method, *args)
     end
 
@@ -56,10 +58,12 @@ module QC
     # The seconds argument must be an integer. The method and args argument
     # must be in the form described in the documentation for the #enqueue
     # method.
+    # This method returns a hash with the id of the enqueued job.
     def enqueue_in(seconds, method, *args)
       QC.log_yield(:measure => 'queue.enqueue') do
-        s = "INSERT INTO #{TABLE_NAME} (q_name, method, args, scheduled_at)
-             VALUES ($1, $2, $3, now() + interval '#{seconds.to_i} seconds')"
+        s = "INSERT INTO #{QC.table_name} (q_name, method, args, scheduled_at)
+             VALUES ($1, $2, $3, now() + interval '#{seconds.to_i} seconds')
+             RETURNING id"
         conn_adapter.execute(s, name, method, JSON.dump(args))
       end
     end
@@ -85,27 +89,27 @@ module QC
 
     def unlock(id)
       QC.log_yield(:measure => 'queue.unlock') do
-        s = "UPDATE #{TABLE_NAME} set locked_at = null where id = $1"
+        s = "UPDATE #{QC.table_name} set locked_at = null where id = $1"
         conn_adapter.execute(s, id)
       end
     end
 
     def delete(id)
       QC.log_yield(:measure => 'queue.delete') do
-        conn_adapter.execute("DELETE FROM #{TABLE_NAME} where id = $1", id)
+        conn_adapter.execute("DELETE FROM #{QC.table_name} where id = $1", id)
       end
     end
 
     def delete_all
       QC.log_yield(:measure => 'queue.delete_all') do
-        s = "DELETE FROM #{TABLE_NAME} WHERE q_name = $1"
+        s = "DELETE FROM #{QC.table_name} WHERE q_name = $1"
         conn_adapter.execute(s, name)
       end
     end
 
     def count
       QC.log_yield(:measure => 'queue.count') do
-        s = "SELECT COUNT(*) FROM #{TABLE_NAME} WHERE q_name = $1"
+        s = "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1"
         r = conn_adapter.execute(s, name)
         r["count"].to_i
       end

data/lib/queue_classic/tasks.rb

@@ -8,7 +8,7 @@ end
 namespace :qc do
   desc "Start a new worker for the (default or $QUEUE / $QUEUES) queue"
   task :work  => :environment do
-    @worker = QC::Worker.new
+    @worker = QC.default_worker_class.new
 
     trap('INT') do
       $stderr.puts("Received INT. Shutting down.")

data/lib/queue_classic/version.rb

@@ -0,0 +1,3 @@
+module QC
+  VERSION = "3.2.0.RC1"
+end

data/lib/queue_classic/worker.rb

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 require_relative 'queue'
 require_relative 'conn_adapter'
 
@@ -16,8 +17,8 @@ module QC
     # q_names:: Names of queues to process. Will process left to right.
     # top_bound:: Offset to the head of the queue. 1 == strict FIFO.
     def initialize(args={})
-      @fork_worker = args[:fork_worker] || QC::FORK_WORKER
-      @wait_interval = args[:wait_interval] || QC::WAIT_TIME
+      @fork_worker = args[:fork_worker] || QC.fork_worker?
+      @wait_interval = args[:wait_interval] || QC.wait_time
 
       if args[:connection]
         @conn_adapter = ConnAdapter.new(args[:connection])
@@ -26,9 +27,9 @@ module QC
       end
 
       @queues = setup_queues(@conn_adapter,
-        (args[:q_name] || QC::QUEUE),
-        (args[:q_names] || QC::QUEUES),
-        (args[:top_bound] || QC::TOP_BOUND))
+        (args[:q_name] || QC.queue),
+        (args[:q_names] || QC.queues),
+        (args[:top_bound] || QC.top_bound))
       log(args.merge(:at => "worker_initialized"))
       @running = true
     end
@@ -101,7 +102,7 @@ module QC
     # then it is deleted from the queue.
     # If the job has raised an exception the responsibility of what
     # to do with the job is delegated to Worker#handle_failure.
-    # If the job is not finished and an INT signal is traped,
+    # If the job is not finished and an INT signal is trapped,
     # this method will unlock the job in the queue.
     def process(queue, job)
       start = Time.now
@@ -136,7 +137,7 @@ module QC
     # This method will be called when an exception
     # is raised during the execution of the job.
     def handle_failure(job,e)
-      $stderr.puts("count#qc.job-error=1 job=#{job} error=#{e.inspect}")
+      $stderr.puts("count#qc.job-error=1 job=#{job} error=#{e.inspect} at=#{e.backtrace.first}")
     end
 
     # This method should be overriden if

data/queue_classic.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'queue_classic/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "queue_classic"
+  spec.email         = "r@32k.io"
+  spec.version       = QC::VERSION
+  spec.description   = "queue_classic is a queueing library for Ruby apps. (Rails, Sinatra, Etc...) queue_classic features asynchronous job polling, database maintained locks and no ridiculous dependencies. As a matter of fact, queue_classic only requires pg."
+  spec.summary       = "Simple, efficient worker queue for Ruby & PostgreSQL."
+  spec.authors       = ["Ryan Smith (♠ ace hacker)"]
+  spec.homepage      = "http://github.com/QueueClassic/queue_classic"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.require_paths = %w[lib]
+
+  spec.add_dependency "pg", ">= 0.17", "< 0.19"
+end

data/sql/create_table.sql

@@ -11,10 +11,12 @@ CREATE TABLE queue_classic_jobs (
   scheduled_at timestamptz default now()
 );
 
--- If json type is available, use it for the args column.
-perform * from pg_type where typname = 'json';
-if found then
-  alter table queue_classic_jobs alter column args type json using (args::json);
+-- If jsonb type is available, use it for the args column
+if exists (select 1 from pg_type where typname = 'jsonb') then
+  alter table queue_classic_jobs alter column args type jsonb using args::jsonb;
+-- Otherwise, use json type for the args column if available
+elsif exists (select 1 from pg_type where typname = 'json') then
+  alter table queue_classic_jobs alter column args type json using args::json;
 end if;
 
 end $$ language plpgsql;

data/test/benchmark_test.rb

@@ -1,37 +1,38 @@
-require File.expand_path("../helper.rb", __FILE__)
+require_relative 'helper'
 
 if ENV["QC_BENCHMARK"]
   class BenchmarkTest < QCTest
+    BENCHMARK_SIZE = Integer(ENV.fetch("QC_BENCHMARK_SIZE", 10_000))
+    BENCHMARK_MAX_TIME_DEQUEUE = Integer(ENV.fetch("QC_BENCHMARK_MAX_TIME_DEQUEUE", 30))
+    BENCHMARK_MAX_TIME_ENQUEUE = Integer(ENV.fetch("QC_BENCHMARK_MAX_TIME_ENQUEUE", 5))
 
     def test_enqueue
-      n = 10_000
       start = Time.now
-      n.times do
-        QC.enqueue("1.odd?", [])
+      BENCHMARK_SIZE.times do
+        QC.enqueue("1.odd?")
       end
-      assert_equal(n, QC.count)
+      assert_equal(BENCHMARK_SIZE, QC.count)
 
       elapsed = Time.now - start
-      assert_in_delta(4, elapsed, 1)
+      assert_operator(elapsed, :<, BENCHMARK_MAX_TIME_ENQUEUE)
     end
 
     def test_dequeue
       worker = QC::Worker.new
       worker.running = true
-      n = 10_000
-      n.times do
-        QC.enqueue("1.odd?", [])
+      BENCHMARK_SIZE.times do
+        QC.enqueue("1.odd?")
       end
-      assert_equal(n, QC.count)
+      assert_equal(BENCHMARK_SIZE, QC.count)
 
       start = Time.now
-      n.times do
+      BENCHMARK_SIZE.times do
         worker.work
       end
       elapsed = Time.now - start
 
       assert_equal(0, QC.count)
-      assert_in_delta(10, elapsed, 3)
+      assert_operator(elapsed, :<, BENCHMARK_MAX_TIME_DEQUEUE)
     end
 
   end

data/test/config_test.rb

@@ -0,0 +1,121 @@
+require_relative 'helper'
+
+class ConfigTest < QCTest
+  def setup
+    QC.reset_config
+  end
+
+  def teardown
+    QC.reset_config
+  end
+
+  def test_app_name_default
+    assert_equal "queue_classic", QC.app_name
+  end
+
+  def test_configure_app_name_with_env_var
+    with_env "QC_APP_NAME" => "zomg_qc" do
+      assert_equal "zomg_qc", QC.app_name
+    end
+  end
+
+  def test_wait_time_default
+    assert_equal 5, QC.wait_time
+  end
+
+  def test_configure_wait_time_with_env_var
+    with_env "QC_LISTEN_TIME" => "7" do
+      assert_equal 7, QC.wait_time
+    end
+  end
+
+  def test_table_name_default
+    assert_equal "queue_classic_jobs", QC.table_name
+  end
+
+  def test_queue_default
+    assert_equal "default", QC.queue
+    assert_equal "default", QC.default_queue.name
+  end
+
+  def test_configure_queue_with_env_var
+    with_env "QUEUE" => "priority" do
+      assert_equal "priority", QC.queue
+      assert_equal "priority", QC.default_queue.name
+    end
+  end
+
+  def test_assign_default_queue
+    QC.default_queue = QC::Queue.new "dispensable"
+    assert_equal "default", QC.queue
+    assert_equal "dispensable", QC.default_queue.name
+  end
+
+  def test_queues_default
+    assert_equal [], QC.queues
+  end
+
+  def test_configure_queues_with_env_var
+    with_env "QUEUES" => "first,second,third" do
+      assert_equal %w(first second third), QC.queues
+    end
+  end
+
+  def test_configure_queues_with_whitespace
+    with_env "QUEUES" => " one, two, three " do
+      assert_equal %w(one two three), QC.queues
+    end
+  end
+
+  def test_top_bound_default
+    assert_equal 9, QC.top_bound
+  end
+
+  def test_configure_top_bound_with_env_var
+    with_env "QC_TOP_BOUND" => "5" do
+      assert_equal 5, QC.top_bound
+    end
+  end
+
+  def test_fork_worker_default
+    refute QC.fork_worker?
+  end
+
+  def test_configure_fork_worker_with_env_var
+    with_env "QC_FORK_WORKER" => "yo" do
+      assert QC.fork_worker?
+    end
+  end
+
+  def test_configuration_constants_are_deprecated
+    warning = capture_stderr_output do
+      QC::FORK_WORKER
+    end
+    assert_match "QC::FORK_WORKER is deprecated", warning
+    assert_match "QC.fork_worker? instead", warning
+  end
+
+  class TestWorker < QC::Worker; end
+
+  def test_default_worker_class
+    assert_equal QC::Worker, QC.default_worker_class
+  end
+
+  def test_configure_default_worker_class_with_env_var
+    if RUBY_VERSION =~ /^1\.9\./
+        skip "Kernel.const_get in Ruby 1.9.x does not perform recursive lookups"
+    end
+    with_env "QC_DEFAULT_WORKER_CLASS" => "ConfigTest::TestWorker" do
+      assert_equal TestWorker, QC.default_worker_class
+    end
+  end
+
+  def test_assign_default_worker_class
+    original_worker = QC.default_worker_class
+    QC.default_worker_class = TestWorker
+
+    assert_equal TestWorker, QC.default_worker_class
+  ensure
+    QC.default_worker_class = original_worker
+  end
+end