queue_classic 2.2.0 → 2.2.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3137935dc373e4d8af179e93422961d1a5c07e89
+  data.tar.gz: 709e4565ef2dcf4635094391558942c02c44e113
+SHA512:
+  metadata.gz: 0141f28277f3f8e26422c77c0787f3cbc84986504a97a762a62119a0036237feea1fc43ee7cc081e2483737d524aa3576d553a1a8601bf15688cf11fc0576f81
+  data.tar.gz: 4abb6f06e3ab1e7ad39e410cea0ff277a4b0948faf923797cec4e193131f1c54d69b36a90fba06c43b9ef6f0ed1f69ad7a9acde266c48f3566fe3a72f9e05ed1

data/lib/generators/queue_classic/install_generator.rb

@@ -0,0 +1,22 @@
+require 'rails/generators'
+require 'rails/generators/migration'
+require 'active_record'
+
+module QC
+  class InstallGenerator < Rails::Generators::Base
+    include Rails::Generators::Migration
+
+    namespace "queue_classic:install"
+    self.source_paths << File.join(File.dirname(__FILE__), 'templates')
+    desc 'Generates (but does not run) a migration to add a queue_classic table.'
+
+    def self.next_migration_number(dirname)
+      next_migration_number = current_migration_number(dirname) + 1
+      ActiveRecord::Migration.next_migration_number(next_migration_number)
+    end
+
+    def create_migration_file
+      migration_template 'add_queue_classic.rb', 'db/migrate/add_queue_classic.rb'
+    end
+  end
+end

data/lib/generators/queue_classic/templates/add_queue_classic.rb

@@ -0,0 +1,9 @@
+class AddQueueClassic < ActiveRecord::Migration
+  def self.up
+    QC::Setup.create
+  end
+
+  def self.down
+    QC::Setup.drop
+  end
+end

data/lib/queue_classic.rb

@@ -1,24 +1,12 @@
-require "pg"
-require "uri"
-require "json"
-
-require "queue_classic/conn"
-require "queue_classic/queries"
-require "queue_classic/queue"
-require "queue_classic/worker"
-require "queue_classic/setup"
-
 module QC
-  Root = File.expand_path("..", File.dirname(__FILE__))
-  SqlFunctions = File.join(QC::Root, "/sql/ddl.sql")
-  DropSqlFunctions = File.join(QC::Root, "/sql/drop_ddl.sql")
-  CreateTable = File.join(QC::Root, "/sql/create_table.sql")
-
   # You can use the APP_NAME to query for
   # postgres related process information in the
   # pg_stat_activity table.
   APP_NAME = ENV["QC_APP_NAME"] || "queue_classic"
 
+  # Number of seconds to block on the listen chanel for new jobs.
+  WAIT_TIME = (ENV["QC_LISTEN_TIME"] || 5).to_i
+
   # Why do you want to change the table name?
   # Just deal with the default OK?
   # If you do want to change this, you will
@@ -35,11 +23,6 @@ module QC
   # There is nothing special about 9....
   TOP_BOUND = (ENV["QC_TOP_BOUND"] || 9).to_i
 
-  # If you are using PostgreSQL > 9
-  # then you will have access to listen/notify with payload.
-  # Set this value if you wish to make your worker more efficient.
-  LISTENING_WORKER = !ENV["QC_LISTENING_WORKER"].nil?
-
   # Set this variable if you wish for
   # the worker to fork a UNIX process for
   # each locked job. Remember to re-establish
@@ -47,11 +30,6 @@ module QC
   # for more details.
   FORK_WORKER = !ENV["QC_FORK_WORKER"].nil?
 
-  # The worker uses an exponential back-off
-  # algorithm to lock a job. This value will be used
-  # as the max exponent.
-  MAX_LOCK_ATTEMPTS = (ENV["QC_MAX_LOCK_ATTEMPTS"] || 5).to_i
-
   # Defer method calls on the QC module to the
   # default queue. This facilitates QC.enqueue()
   def self.method_missing(sym, *args, &block)
@@ -101,3 +79,8 @@ module QC
     return result
   end
 end
+
+require "queue_classic/queue"
+require "queue_classic/worker"
+require "queue_classic/setup"
+require "queue_classic/railtie" if defined?(Rails)

data/lib/queue_classic/conn.rb

@@ -1,4 +1,6 @@
 require 'thread'
+require 'uri'
+require 'pg'
 
 module QC
   module Conn
@@ -22,15 +24,10 @@ module QC
       end
     end
 
-    def notify(chan)
-      log(:at => "NOTIFY")
-      execute('NOTIFY "' + chan + '"') #quotes matter
-    end
-
-    def wait(chan, t)
-      listen(chan)
-      wait_for_notify(t)
-      unlisten(chan)
+    def wait(chan)
+      execute('LISTEN "' + chan + '"')
+      wait_for_notify(WAIT_TIME)
+      execute('UNLISTEN "' + chan + '"')
       drain_notify
     end
 
@@ -106,19 +103,9 @@ module QC
       QC.log(msg)
     end
 
-    def listen(chan)
-      log(:at => "LISTEN")
-      execute('LISTEN "' + chan + '"') #quotes matter
-    end
-
-    def unlisten(chan)
-      log(:at => "UNLISTEN")
-      execute('UNLISTEN "' + chan + '"') #quotes matter
-    end
-
     def wait_for_notify(t)
-      connection.wait_for_notify(t) do |event, pid, msg|
-        log(:at => "received_notification")
+      Array.new.tap do |msgs|
+        connection.wait_for_notify(t) {|event, pid, msg| msgs << msg}
       end
     end
 

data/lib/queue_classic/queue.rb

@@ -1,30 +1,53 @@
+require 'queue_classic'
+require 'queue_classic/conn'
+require 'json'
+
 module QC
   class Queue
 
-    attr_reader :name, :chan
-    def initialize(name, notify=QC::LISTENING_WORKER)
+    attr_reader :name, :top_bound
+    def initialize(name, top_bound=nil)
       @name = name
-      @chan = @name if notify
+      @top_bound = top_bound || QC::TOP_BOUND
     end
 
     def enqueue(method, *args)
-      Queries.insert(name, method, args, chan)
+      QC.log_yield(:measure => 'queue.enqueue') do
+        s="INSERT INTO #{TABLE_NAME} (q_name, method, args) VALUES ($1, $2, $3)"
+        res = Conn.execute(s, name, method, JSON.dump(args))
+      end
     end
 
-    def lock(top_bound=TOP_BOUND)
-      Queries.lock_head(name, top_bound)
+    def lock
+      QC.log_yield(:measure => 'queue.lock') do
+        s = "SELECT * FROM lock_head($1, $2)"
+        if r = Conn.execute(s, name, top_bound)
+          {:id => r["id"],
+            :method => r["method"],
+            :args => JSON.parse(r["args"])}
+        end
+      end
     end
 
     def delete(id)
-      Queries.delete(id)
+      QC.log_yield(:measure => 'queue.delete') do
+        Conn.execute("DELETE FROM #{TABLE_NAME} where id = $1", id)
+      end
     end
 
     def delete_all
-      Queries.delete_all(name)
+      QC.log_yield(:measure => 'queue.delete_all') do
+        s = "DELETE FROM #{TABLE_NAME} WHERE q_name = $1"
+        Conn.execute(s, name)
+      end
     end
 
     def count
-      Queries.count(name)
+      QC.log_yield(:measure => 'queue.count') do
+        s = "SELECT COUNT(*) FROM #{TABLE_NAME} WHERE q_name = $1"
+        r = Conn.execute(s, name)
+        r["count"].to_i
+      end
     end
 
   end

data/lib/queue_classic/railtie.rb

@@ -0,0 +1,9 @@
+require 'rails/railtie'
+
+module QC
+  class Railtie < ::Rails::Railtie
+    rake_tasks do
+      load 'queue_classic/tasks.rb'
+    end
+  end
+end

data/lib/queue_classic/setup.rb

@@ -1,40 +1,18 @@
 module QC
   module Setup
-    extend self
+    Root = File.expand_path("../..", File.dirname(__FILE__))
+    SqlFunctions = File.join(Root, "/sql/ddl.sql")
+    CreateTable = File.join(Root, "/sql/create_table.sql")
+    DropSqlFunctions = File.join(Root, "/sql/drop_ddl.sql")
 
-    def create
-      create_table
-      create_functions
+    def self.create
+      Conn.execute(File.read(CreateTable))
+      Conn.execute(File.read(SqlFunctions))
     end
 
-    def drop
-      drop_functions
-      drop_table
+    def self.drop
+      Conn.execute("DROP TABLE IF EXISTS queue_classic_jobs CASCADE")
+      Conn.execute(File.read(DropSqlFunctions))
     end
-
-    def create_table
-      Conn.transaction do
-        Conn.execute(File.read(CreateTable))
-      end
-    end
-
-    def drop_table
-      Conn.transaction do
-        Conn.execute("DROP TABLE IF EXISTS queue_classic_jobs")
-      end
-    end
-
-    def create_functions
-      Conn.transaction do
-        Conn.execute(File.read(SqlFunctions))
-      end
-    end
-
-    def drop_functions
-      Conn.transaction do
-        Conn.execute(File.read(DropSqlFunctions))
-      end
-    end
-
   end
 end

data/lib/queue_classic/worker.rb

@@ -1,3 +1,7 @@
+require 'queue_classic'
+require 'queue_classic/queue'
+require 'queue_classic/conn'
+
 module QC
   class Worker
 
@@ -5,15 +9,10 @@ module QC
     # In the case no arguments are passed to the initializer,
     # the defaults are pulled from the environment variables.
     def initialize(args={})
-      @q_name           = args[:q_name]           ||= QC::QUEUE
-      @top_bound        = args[:top_bound]        ||= QC::TOP_BOUND
-      @fork_worker      = args[:fork_worker]      ||= QC::FORK_WORKER
-      @listening_worker = args[:listening_worker] ||= QC::LISTENING_WORKER
-      @max_attempts     = args[:max_attempts]     ||= QC::MAX_LOCK_ATTEMPTS
-
-      @running = true
-      @queue = Queue.new(@q_name, @listening_worker)
+      @fork_worker = args[:fork_worker] || QC::FORK_WORKER
+      @queue = QC::Queue.new((args[:q_name] || QC::QUEUE), args[:top_bound])
       log(args.merge(:at => "worker_initialized"))
+      @running = true
     end
 
     # Start a loop and work jobs indefinitely.
@@ -36,9 +35,9 @@ module QC
     # Define setup_child to hook into the forking process.
     # Using setup_child is good for re-establishing database connections.
     def fork_and_work
-      @cpid = fork {setup_child; work}
-      log(:at => :fork, :pid => @cpid)
-      Process.wait(@cpid)
+      cpid = fork {setup_child; work}
+      log(:at => :fork, :pid => cpid)
+      Process.wait(cpid)
     end
 
     # This method will lock a job & process the job.
@@ -50,33 +49,15 @@ module QC
       end
     end
 
-    # lock_job will attempt to lock a job in the queue's table. It uses an
-    # exponential backoff in the event that a job was not locked. This method
-    # will return a hash when a job is obtained.
-    #
-    # This method will terminate early if the stop method is called or
-    # @max_attempts has been reached.
-    #
-    # It is important that callers delete the job when finished.
-    # *@queue.delete(job[:id])*
+    # Attempt to lock a job in the queue's table.
+    # Return a hash when a job is locked.
+    # Caller responsible for deleting the job when finished.
     def lock_job
       log(:at => "lock_job")
-      attempts = 0
       job = nil
-      until !@running || job
-        job = @queue.lock(@top_bound)
-        if job.nil?
-          log(:at => "failed_lock", :attempts => attempts)
-          if attempts < @max_attempts
-            wait(2**attempts)
-            attempts += 1
-            next
-          else
-            break
-          end
-        else
-          log(:at => "finished_lock", :job => job[:id])
-        end
+      while @running
+        break if job = @queue.lock
+        Conn.wait(@queue.name)
       end
       job
     end
@@ -106,21 +87,6 @@ module QC
       klass.send(message, *args)
     end
 
-    # If @listening_worker is set, the worker will use the database
-    # to sleep. The database approach preferred over a syscall since
-    # the database will break the sleep when new jobs are inserted into
-    # the queue.
-    def wait(t)
-      if @listening_worker
-        log(:at => "listen_wait", :wait => t)
-        Conn.wait(@queue.chan)
-      else
-        log(:at => "sleep_wait", :wait => t)
-        Kernel.sleep(t)
-      end
-      log(:at => "finished_listening")
-    end
-
     # This method will be called when an exception
     # is raised during the execution of the job.
     def handle_failure(job,e)

data/readme.md

@@ -1,6 +1,6 @@
 # queue_classic
 
-v2.2.0
+v2.2.1
 
 queue_classic provides a simple interface to a PostgreSQL-backed message queue. queue_classic specializes in concurrent locking and minimizing database load while providing a simple, intuitive developer experience. queue_classic assumes that you are already using PostgreSQL in your production environment and that adding another dependency (e.g. redis, beanstalkd, 0mq) is undesirable.
 
@@ -14,7 +14,7 @@ Features:
 
 Contents:
 
-* [Documentation](http://rubydoc.info/gems/queue_classic/2.2.0/frames)
+* [Documentation](http://rubydoc.info/gems/queue_classic/2.2.1/frames)
 * [Usage](#usage)
 * [Setup](#setup)
 * [Configuration](#configuration)
@@ -95,7 +95,7 @@ class MyWorker < QC::Worker
  	end
 end
 
-worker = MyWorker.new(max_attempts: 10, listening_worker: true)
+worker = MyWorker.new
 
 trap('INT') {exit}
 trap('TERM') {worker.stop}
@@ -129,14 +129,7 @@ Declare dependencies in Gemfile.
 
 ```ruby
 source "http://rubygems.org"
-gem "queue_classic", "2.2.0"
-```
-
-Require these files in your Rakefile so that you can run `rake qc:work`.
-
-```ruby
-require "queue_classic"
-require "queue_classic/tasks"
+gem "queue_classic", "2.2.1"
 ```
 
 By default, queue_classic will use the QC_DATABASE_URL falling back on DATABASE_URL. The URL must be in the following format: `postgres://username:password@localhost/database_name`.  If you use Heroku's PostgreSQL service, this will already be set. If you don't want to set this variable, you can set the connection in an initializer. **QueueClassic will maintain its own connection to the database.** This may double the number of connections to your database. Set QC::Conn.connection to share the connection between Rails & QueueClassic
@@ -146,21 +139,15 @@ require 'queue_classic'
 QC::Conn.connection = ActiveRecord::Base.connection.raw_connection
 ```
 
-**Note on using ActiveRecord migrations:** If you use the migration, and you wish to use commands that reset the database from the stored schema (e.g. `rake db:reset`), your application must be configured with `config.active_record.schema_format = :sql` in `config/application.rb`.  If you don't do this, the PL/pgSQL function that queue_classic creates will be lost when you reset the database.
+Next you need to run the queue classic generator to create the database
+migration. This will setup the necessary table to use queue classic.
 
-```ruby
-require 'queue_classic'
-
-class AddQueueClassic < ActiveRecord::Migration
-  def self.up
-    QC::Setup.create
-  end
-
-  def self.down
-    QC::Setup.drop
-  end
-end
 ```
+rails generate queue_classic:install
+rake db:migrate
+```
+
+**Note on using ActiveRecord migrations:** If you use the migration, and you wish to use commands that reset the database from the stored schema (e.g. `rake db:reset`), your application must be configured with `config.active_record.schema_format = :sql` in `config/application.rb`.  If you don't do this, the PL/pgSQL function that queue_classic creates will be lost when you reset the database.
 
 ### Rake Task Setup
 

data/sql/create_table.sql

@@ -16,4 +16,14 @@ end if;
 
 end $$ language plpgsql;
 
+create function queue_classic_notify() returns trigger as $$ begin
+  perform pg_notify(new.q_name, '');
+  return null;
+end $$ language plpgsql;
+
+create trigger queue_classic_notify
+after insert on queue_classic_jobs
+for each row
+execute procedure queue_classic_notify();
+
 CREATE INDEX idx_qc_on_name_only_unlocked ON queue_classic_jobs (q_name, id) WHERE locked_at IS NULL;

data/sql/drop_ddl.sql

@@ -1,2 +1,3 @@
 DROP FUNCTION IF EXISTS lock_head(tname varchar);
-DROP FUNCTION IF EXISTS lock_head(q_name varchar, top_boundary integer)
+DROP FUNCTION IF EXISTS lock_head(q_name varchar, top_boundary integer);
+DROP FUNCTION IF EXISTS queue_classic_notify() cascade;

data/test/helper.rb

@@ -17,36 +17,11 @@ class QCTest < Minitest::Test
     QC.delete_all
   end
 
-  def init_db(table_name="queue_classic_jobs")
+  def init_db
     QC::Conn.execute("SET client_min_messages TO 'warning'")
     QC::Setup.drop
     QC::Setup.create
-    QC::Conn.execute(<<EOS)
-DO $$
--- Set initial sequence to a large number to test the entire toolchain
--- works on integers with higher bits set.
-DECLARE
-    quoted_name text;
-    quoted_size text;
-BEGIN
-    -- Find the name of the relevant sequence.
-    --
-    -- pg_get_serial_sequence quotes identifiers as part of its
-    -- behavior.
-    SELECT name
-    INTO STRICT quoted_name
-    FROM pg_get_serial_sequence('queue_classic_jobs', 'id') AS name;
-
-    -- Don't quote, because ALTER SEQUENCE RESTART doesn't like
-    -- general literals, only unquoted numeric literals.
-    SELECT pow(2, 34)::text AS size
-    INTO STRICT quoted_size;
-
-    EXECUTE 'ALTER SEQUENCE ' || quoted_name ||
-        ' RESTART ' || quoted_size || ';';
-END;
-$$;
-EOS
+    QC::Conn.execute(File.read('./test/helper.sql'))
   end
 
   def capture_debug_output

data/test/queue_test.rb

@@ -56,7 +56,7 @@ class QueueTest < QCTest
   end
 
   def test_queue_instance
-    queue = QC::Queue.new("queue_classic_jobs", false)
+    queue = QC::Queue.new("queue_classic_jobs")
     queue.enqueue("Klass.method")
     assert_equal(1, queue.count)
     queue.delete(queue.lock[:id])
@@ -64,7 +64,7 @@ class QueueTest < QCTest
   end
 
   def test_repair_after_error
-    queue = QC::Queue.new("queue_classic_jobs", false)
+    queue = QC::Queue.new("queue_classic_jobs")
     queue.enqueue("Klass.method")
     assert_equal(1, queue.count)
     connection = QC::Conn.connection
@@ -100,4 +100,17 @@ class QueueTest < QCTest
   ensure
     QC.default_queue = nil
   end
+
+  def test_enqueue_triggers_notify
+    QC::Conn.execute('LISTEN "' + QC::QUEUE + '"')
+    QC::Conn.send(:drain_notify)
+
+    msgs = QC::Conn.send(:wait_for_notify, 0.25)
+    assert_equal(0, msgs.length)
+
+    QC.enqueue("Klass.method")
+    msgs = QC::Conn.send(:wait_for_notify, 0.25)
+    assert_equal(1, msgs.length)
+  end
+
 end

metadata

@@ -1,8 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: queue_classic
 version: !ruby/object:Gem::Version
-  version: 2.2.0
-  prerelease: 
+  version: 2.2.1
 platform: ruby
 authors:
 - Ryan Smith (♠ ace hacker)
@@ -14,7 +13,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -39,9 +36,11 @@ files:
 - sql/create_table.sql
 - sql/ddl.sql
 - sql/drop_ddl.sql
+- lib/generators/queue_classic/install_generator.rb
+- lib/generators/queue_classic/templates/add_queue_classic.rb
 - lib/queue_classic/conn.rb
-- lib/queue_classic/queries.rb
 - lib/queue_classic/queue.rb
+- lib/queue_classic/railtie.rb
 - lib/queue_classic/setup.rb
 - lib/queue_classic/tasks.rb
 - lib/queue_classic/worker.rb
@@ -54,27 +53,26 @@ files:
 homepage: http://github.com/ryandotsmith/queue_classic
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: postgres backed queue
 test_files:
 - test/benchmark_test.rb

data/lib/queue_classic/queries.rb

@@ -1,42 +0,0 @@
-module QC
-  module Queries
-    extend self
-
-    def insert(q_name, method, args, chan=nil)
-      QC.log_yield(:action => "insert_job") do
-        s = "INSERT INTO #{TABLE_NAME} (q_name, method, args) VALUES ($1, $2, $3)"
-        res = Conn.execute(s, q_name, method, JSON.dump(args))
-        Conn.notify(chan) if chan
-      end
-    end
-
-    def lock_head(q_name, top_bound)
-      s = "SELECT * FROM lock_head($1, $2)"
-      if r = Conn.execute(s, q_name, top_bound)
-        {
-          :id     => r["id"],
-          :method => r["method"],
-          :args   => JSON.parse(r["args"])
-        }
-      end
-    end
-
-    def count(q_name=nil)
-      s = "SELECT COUNT(*) FROM #{TABLE_NAME}"
-      s << " WHERE q_name = $1" if q_name
-      r = Conn.execute(*[s, q_name].compact)
-      r["count"].to_i
-    end
-
-    def delete(id)
-      Conn.execute("DELETE FROM #{TABLE_NAME} where id = $1", id)
-    end
-
-    def delete_all(q_name=nil)
-      s = "DELETE FROM #{TABLE_NAME}"
-      s << " WHERE q_name = $1" if q_name
-      Conn.execute(*[s, q_name].compact)
-    end
-
-  end
-end