queue_classic 2.1.4 → 2.2.0

data/lib/queue_classic.rb

@@ -1,6 +1,6 @@
 require "pg"
 require "uri"
-require "multi_json"
+require "json"
 
 require "queue_classic/conn"
 require "queue_classic/queries"
@@ -16,11 +16,8 @@ module QC
 
   # You can use the APP_NAME to query for
   # postgres related process information in the
-  # pg_stat_activity table. Don't set this unless
-  # you are using PostgreSQL > 9.0
-  if APP_NAME = ENV["QC_APP_NAME"]
-    Conn.execute("SET application_name = '#{APP_NAME}'")
-  end
+  # pg_stat_activity table.
+  APP_NAME = ENV["QC_APP_NAME"] || "queue_classic"
 
   # Why do you want to change the table name?
   # Just deal with the default OK?
@@ -66,6 +63,10 @@ module QC
     default_queue.respond_to?(method_name)
   end
 
+  def self.default_queue=(queue)
+    @default_queue = queue
+  end
+
   def self.default_queue
     @default_queue ||= begin
       Queue.new(QUEUE)
@@ -91,7 +92,7 @@ module QC
     if block_given?
       start = Time.now
       result = yield
-      data.merge(:elapsed => Time.now - start)
+      data.merge(:elapsed => Integer((Time.now - t0)*1000))
     end
     data.reduce(out=String.new) do |s, tup|
       s << [tup.first, tup.last].join("=") << " "
@@ -99,5 +100,4 @@ module QC
     puts(out) if ENV["DEBUG"]
     return result
   end
-
 end

data/lib/queue_classic/conn.rb

@@ -27,26 +27,11 @@ module QC
       execute('NOTIFY "' + chan + '"') #quotes matter
     end
 
-    def listen(chan)
-      log(:at => "LISTEN")
-      execute('LISTEN "' + chan + '"') #quotes matter
-    end
-
-    def unlisten(chan)
-      log(:at => "UNLISTEN")
-      execute('UNLISTEN "' + chan + '"') #quotes matter
-    end
-
-    def drain_notify
-      until connection.notifies.nil?
-        log(:at => "drain_notifications")
-      end
-    end
-
-    def wait_for_notify(t)
-      connection.wait_for_notify(t) do |event, pid, msg|
-        log(:at => "received_notification")
-      end
+    def wait(chan, t)
+      listen(chan)
+      wait_for_notify(t)
+      unlisten(chan)
+      drain_notify
     end
 
     def transaction
@@ -69,7 +54,7 @@ module QC
     end
 
     def connection=(connection)
-      unless connection.instance_of? PG::Connection
+      unless connection.is_a? PG::Connection
         c = connection.class
         err = "connection must be an instance of PG::Connection, but was #{c}"
         raise(ArgumentError, err)
@@ -85,20 +70,28 @@ module QC
 
     def connect
       log(:at => "establish_conn")
-      conn = PGconn.connect(
-        db_url.host.gsub(/%2F/i, '/'), # host or percent-encoded socket path
-        db_url.port || 5432,
-        nil, '', #opts, tty
-        db_url.path.gsub("/",""), # database name
-        db_url.user,
-        db_url.password
-      )
+      conn = PGconn.connect(*normalize_db_url(db_url))
       if conn.status != PGconn::CONNECTION_OK
         log(:error => conn.error)
       end
+      conn.exec("SET application_name = '#{QC::APP_NAME}'")
       conn
     end
 
+    def normalize_db_url(url)
+      host = url.host
+      host = host.gsub(/%2F/i, '/') if host
+
+      [
+       host, # host or percent-encoded socket path
+       url.port || 5432,
+       nil, '', #opts, tty
+       url.path.gsub("/",""), # database name
+       url.user,
+       url.password
+      ]
+    end
+
     def db_url
       return @db_url if @db_url
       url = ENV["QC_DATABASE_URL"] ||
@@ -107,9 +100,33 @@ module QC
       @db_url = URI.parse(url)
     end
 
+    private
+
     def log(msg)
       QC.log(msg)
     end
 
+    def listen(chan)
+      log(:at => "LISTEN")
+      execute('LISTEN "' + chan + '"') #quotes matter
+    end
+
+    def unlisten(chan)
+      log(:at => "UNLISTEN")
+      execute('UNLISTEN "' + chan + '"') #quotes matter
+    end
+
+    def wait_for_notify(t)
+      connection.wait_for_notify(t) do |event, pid, msg|
+        log(:at => "received_notification")
+      end
+    end
+
+    def drain_notify
+      until connection.notifies.nil?
+        log(:at => "drain_notifications")
+      end
+    end
+
   end
 end

data/lib/queue_classic/queries.rb

@@ -5,7 +5,7 @@ module QC
     def insert(q_name, method, args, chan=nil)
       QC.log_yield(:action => "insert_job") do
         s = "INSERT INTO #{TABLE_NAME} (q_name, method, args) VALUES ($1, $2, $3)"
-        res = Conn.execute(s, q_name, method, MultiJson.encode(args))
+        res = Conn.execute(s, q_name, method, JSON.dump(args))
         Conn.notify(chan) if chan
       end
     end
@@ -16,7 +16,7 @@ module QC
         {
           :id     => r["id"],
           :method => r["method"],
-          :args   => MultiJson.decode(r["args"])
+          :args   => JSON.parse(r["args"])
         }
       end
     end

data/lib/queue_classic/tasks.rb

@@ -19,7 +19,7 @@ namespace :qc do
     puts QC::Worker.new.queue.count
   end
 
-  desc "Setup queue_classic tables and funtions in database"
+  desc "Setup queue_classic tables and functions in database"
   task :create => :environment do
     QC::Setup.create
   end

data/lib/queue_classic/worker.rb

@@ -1,7 +1,7 @@
 module QC
   class Worker
 
-    attr_reader :queue
+    attr_accessor :queue, :running
     # In the case no arguments are passed to the initializer,
     # the defaults are pulled from the environment variables.
     def initialize(args={})
@@ -113,21 +113,18 @@ module QC
     def wait(t)
       if @listening_worker
         log(:at => "listen_wait", :wait => t)
-        Conn.listen(@queue.chan)
-        Conn.wait_for_notify(t)
-        Conn.unlisten(@queue.chan)
-        Conn.drain_notify
-        log(:at => "finished_listening")
+        Conn.wait(@queue.chan)
       else
         log(:at => "sleep_wait", :wait => t)
         Kernel.sleep(t)
       end
+      log(:at => "finished_listening")
     end
 
     # This method will be called when an exception
     # is raised during the execution of the job.
     def handle_failure(job,e)
-      log(:at => "handle_failure")
+      log(:at => "handle_failure", :job => job, :error => e.inspect)
     end
 
     # This method should be overriden if

data/readme.md

@@ -1,6 +1,6 @@
 # queue_classic
 
-v2.1.4
+v2.2.0
 
 queue_classic provides a simple interface to a PostgreSQL-backed message queue. queue_classic specializes in concurrent locking and minimizing database load while providing a simple, intuitive developer experience. queue_classic assumes that you are already using PostgreSQL in your production environment and that adding another dependency (e.g. redis, beanstalkd, 0mq) is undesirable.
 
@@ -14,10 +14,11 @@ Features:
 
 Contents:
 
-* [Documentation](http://rubydoc.info/gems/queue_classic/2.1.4/frames)
+* [Documentation](http://rubydoc.info/gems/queue_classic/2.2.0/frames)
 * [Usage](#usage)
 * [Setup](#setup)
 * [Configuration](#configuration)
+* [Support](#support)
 * [Hacking](#hacking-on-queue_classic)
 * [License](#license)
 
@@ -30,6 +31,8 @@ There are 2 ways to use queue_classic.
 
 ### Producing Jobs
 
+The first argument is a string which represents a ruby object and a method name. The second argument(s) will be passed along as arguments to the method invocation defined by the first argument. The set of arguments will be encoded as JSON in the database.
+
 ```ruby
 # This method has no arguments.
 QC.enqueue("Time.now")
@@ -51,12 +54,6 @@ p_queue = QC::Queue.new("priority_queue")
 p_queue.enqueue("Kernel.puts", ["hello", "world"])
 ```
 
-QueueClassic uses [MultiJSON](https://github.com/intridea/multi_json) to encode the job's payload.
-
-```ruby
-MultiJson.dump({"test" => "test"})
-```
-
 ### Working Jobs
 
 There are two ways to work jobs. The first approach is to use the Rake task. The second approach is to use a custom executable.
@@ -90,9 +87,6 @@ This example is probably not production ready; however, it serves as an example
 require 'timeout'
 require 'queue_classic'
 
-trap('INT') {exit}
-trap('TERM') {worker.stop}
-
 FailedQueue = QC::Queue.new("failed_jobs")
 
 class MyWorker < QC::Worker
@@ -103,6 +97,9 @@ end
 
 worker = MyWorker.new(max_attempts: 10, listening_worker: true)
 
+trap('INT') {exit}
+trap('TERM') {worker.stop}
+
 loop do
 	job = worker.lock_job
 	Thread.new do
@@ -132,7 +129,7 @@ Declare dependencies in Gemfile.
 
 ```ruby
 source "http://rubygems.org"
-gem "queue_classic", "2.1.4"
+gem "queue_classic", "2.2.0"
 ```
 
 Require these files in your Rakefile so that you can run `rake qc:work`.
@@ -181,6 +178,13 @@ $ bundle exec rake qc:drop
 
 All configuration takes place in the form of environment vars. See [queue_classic.rb](https://github.com/ryandotsmith/queue_classic/blob/master/lib/queue_classic.rb#L23-62) for a list of options.
 
+## JSON
+
+If you are running PostgreSQL 9.2 or higher, queue_classic will use the [json](http://www.postgresql.org/docs/9.2/static/datatype-json.html) datatype for storing arguments. Versions 9.1 and lower will use the 'text' column. If you have installed queue_classic prior to version 2.1.4 and are running PostgreSQL >= 9.2, run the following to switch to using the json type:
+```
+alter table queue_classic_jobs alter column args type json using (args::json);
+```
+
 ## Logging
 
 By default queue_classic does not talk very much.
@@ -191,8 +195,23 @@ you can enable the debug output by setting the `DEBUG` environment variable:
 export DEBUG="true"
 ```
 
+## Support
+
+If you think you have found a bug, feel free to open an issue. Use the following template for the new issue:
+
+1. List Versions: Ruby, PostgreSQL, queue_classic.
+2. Define what you would have expcted to happen.
+3. List what actually happened.
+4. Provide sample codes & commands which will reproduce the problem.
+
+If you have general questions about how to use queue_classic, send a message to the mailing list:
+
+https://groups.google.com/d/forum/queue_classic
+
 ## Hacking on queue_classic
 
+[![Build Status](https://drone.io/github.com/ryandotsmith/queue_classic/status.png)](https://drone.io/github.com/ryandotsmith/queue_classic/latest)
+
 ### Dependencies
 
 * Ruby 1.9.2 (tests work in 1.8.7 but compatibility is not guaranteed or supported)

data/sql/create_table.sql

@@ -1,9 +1,19 @@
+do $$ begin
+
 CREATE TABLE queue_classic_jobs (
   id bigserial PRIMARY KEY,
-  q_name varchar(255),
-  method varchar(255),
-  args text,
+  q_name text not null check (length(q_name) > 0),
+  method text not null check (length(method) > 0),
+  args   text not null,
   locked_at timestamptz
 );
 
+-- If json type is available, use it for the args column.
+perform * from pg_type where typname = 'json';
+if found then
+  alter table queue_classic_jobs alter column args type json using (args::json);
+end if;
+
+end $$ language plpgsql;
+
 CREATE INDEX idx_qc_on_name_only_unlocked ON queue_classic_jobs (q_name, id) WHERE locked_at IS NULL;

data/test/benchmark_test.rb

@@ -0,0 +1,38 @@
+require File.expand_path("../helper.rb", __FILE__)
+
+if ENV["QC_BENCHMARK"]
+  class BenchmarkTest < QCTest
+
+    def test_enqueue
+      n = 10_000
+      start = Time.now
+      n.times do
+        QC.enqueue("1.odd?", [])
+      end
+      assert_equal(n, QC.count)
+
+      elapsed = Time.now - start
+      assert_in_delta(4, elapsed, 1)
+    end
+
+    def test_dequeue
+      worker = QC::Worker.new
+      worker.running = true
+      n = 10_000
+      n.times do
+        QC.enqueue("1.odd?", [])
+      end
+      assert_equal(n, QC.count)
+
+      start = Time.now
+      n.times do
+        worker.work
+      end
+      elapsed = Time.now - start
+
+      assert_equal(0, QC.count)
+      assert_in_delta(10, elapsed, 3)
+    end
+
+  end
+end

data/test/conn_test.rb

@@ -0,0 +1,22 @@
+require File.expand_path("../helper.rb", __FILE__)
+
+class ConnTest < QCTest
+
+  def test_extracts_the_segemnts_to_connect
+    database_url = "postgres://ryan:secret@localhost:1234/application_db"
+    normalized = QC::Conn.normalize_db_url(URI.parse(database_url))
+    assert_equal ["localhost",
+                  1234,
+                  nil, "",
+                  "application_db",
+                  "ryan",
+                  "secret"], normalized
+  end
+
+  def test_regression_database_url_without_host
+    database_url = "postgres:///my_db"
+    normalized = QC::Conn.normalize_db_url(URI.parse(database_url))
+    assert_equal [nil, 5432, nil, "", "my_db", nil, nil], normalized
+  end
+
+end

data/test/helper.rb

@@ -4,10 +4,10 @@ $: << File.expand_path("test")
 ENV["DATABASE_URL"] ||= "postgres:///queue_classic_test"
 
 require "queue_classic"
-require "minitest/unit"
-MiniTest::Unit.autorun
+require "stringio"
+require "minitest/autorun"
 
-class QCTest < MiniTest::Unit::TestCase
+class QCTest < Minitest::Test
 
   def setup
     init_db
@@ -47,7 +47,19 @@ BEGIN
 END;
 $$;
 EOS
-    QC::Conn.disconnect
+  end
+
+  def capture_debug_output
+    original_debug = ENV['DEBUG']
+    original_stdout = $stdout
+
+    ENV['DEBUG'] = "true"
+    $stdout = StringIO.new
+    yield
+    $stdout.string
+  ensure
+    ENV['DEBUG'] = original_debug
+    $stdout = original_stdout
   end
 
 end

data/test/queue_test.rb

@@ -72,7 +72,7 @@ class QueueTest < QCTest
     def connection.exec(*args)
       raise PGError
     end
-    assert_raises(PG::Error) { queue.enqueue("Klass.other_method") }    
+    assert_raises(PG::Error) { queue.enqueue("Klass.other_method") }
     assert_equal(1, queue.count)
     queue.enqueue("Klass.other_method")
     assert_equal(2, queue.count)
@@ -80,4 +80,24 @@ class QueueTest < QCTest
     QC::Conn.disconnect
     assert false, "Expected to QC repair after connection error"
   end
+
+  def test_custom_default_queue
+    queue_class = Class.new do
+      attr_accessor :jobs
+      def enqueue(method, *args)
+        @jobs ||= []
+        @jobs << method
+      end
+    end
+
+    queue_instance = queue_class.new
+    QC.default_queue = queue_instance
+
+    QC.enqueue("Klass.method1")
+    QC.enqueue("Klass.method2")
+
+    assert_equal ["Klass.method1", "Klass.method2"], queue_instance.jobs
+  ensure
+    QC.default_queue = nil
+  end
 end

data/test/worker_test.rb

@@ -42,6 +42,33 @@ class WorkerTest < QCTest
     assert_equal(1, worker.failed_count)
   end
 
+  def test_failed_job_is_logged
+    output = capture_debug_output do
+      QC.enqueue("TestObject.not_a_method")
+      QC::Worker.new.work
+    end
+    expected_output = /lib=queue-classic at=handle_failure job={:id=>"\d+", :method=>"TestObject.not_a_method", :args=>\[\]} error=#<NoMethodError: undefined method `not_a_method' for TestObject:Module>/
+    assert_match(expected_output, output, "=== debug output ===\n #{output}")
+  end
+
+  def test_log_yield
+    output = capture_debug_output do
+      QC.log_yield(:action => "test") do
+        0 == 1
+      end
+    end
+    expected_output = /lib=queue-classic action=test elapsed=\d*/
+    assert_match(expected_output, output, "=== debug output ===\n #{output}")
+  end
+
+  def test_log
+    output = capture_debug_output do
+      QC.log(:action => "test")
+    end
+    expected_output = /lib=queue-classic action=test/
+    assert_match(expected_output, output, "=== debug output ===\n #{output}")
+  end
+
   def test_work_with_no_args
     QC.enqueue("TestObject.no_args")
     worker = TestWorker.new

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: queue_classic
 version: !ruby/object:Gem::Version
-  version: 2.1.4
+  version: 2.2.0
   prerelease: 
 platform: ruby
 authors:
@@ -27,22 +27,6 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 0.15.1
-- !ruby/object:Gem::Dependency
-  name: multi_json
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.7.2
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.7.2
 description: queue_classic is a queueing library for Ruby apps. (Rails, Sinatra, Etc...)
   queue_classic features asynchronous job polling, database maintained locks and no
   ridiculous dependencies. As a matter of fact, queue_classic only requires pg.
@@ -62,6 +46,8 @@ files:
 - lib/queue_classic/tasks.rb
 - lib/queue_classic/worker.rb
 - lib/queue_classic.rb
+- test/benchmark_test.rb
+- test/conn_test.rb
 - test/helper.rb
 - test/queue_test.rb
 - test/worker_test.rb
@@ -91,5 +77,7 @@ signing_key:
 specification_version: 3
 summary: postgres backed queue
 test_files:
+- test/benchmark_test.rb
+- test/conn_test.rb
 - test/queue_test.rb
 - test/worker_test.rb