queue_classic 3.0.3 → 3.1.0.RC1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cfade5886b2c70ee4702e8237fd27593f19d752f
-  data.tar.gz: 01102c720913864a3ff4199fe3bd5f85d33ce542
+  metadata.gz: 4280312436762af6532d5ba73b94063e2f3fdfcf
+  data.tar.gz: 8ad63234d7e013a19238e6fbf1722c9eb688af8b
 SHA512:
-  metadata.gz: 93d1dff4a3d75befa7f9d7a4aa89bdee0ee6b6584c434592c01ccf9bbdbed17d91b4906caa13acff26fb302ce02ff8c144cbd2d9897381cc1217b697a73a2db6
-  data.tar.gz: 50c35143f017995c4e9665dd36f8f48b1d796cf26139bc99168e559061ebdb686b43277bee92dc286b3c88256e606cebec191f16bc346c997ece5241eaf2373c
+  metadata.gz: 6b528e1cb1e85ec03e7ccecd36e20394bd578d6e912a8ae531c253ef0ad0c2e996dcb5a53be56650af09c117be67410e3562dcfd37f067b323fb0584c0b6f841
+  data.tar.gz: 02f999edfa7d92a9286b31970e1ef999dc0a37e0355b6f3bd50454b790c154295cfb0db541d9274f3f624486d305c31bbc19db49deb8d026fd69bdf9fcf24603

data/lib/generators/queue_classic/install_generator.rb

@@ -27,6 +27,10 @@ module QC
       if self.class.migration_exists?('db/migrate', 'update_queue_classic_3_0_2').nil?
         migration_template 'update_queue_classic_3_0_2.rb', 'db/migrate/update_queue_classic_3_0_2.rb'
       end
+
+      if self.class.migration_exists?('db/migrate', 'update_queue_classic_3_1_0').nil?
+        migration_template 'update_queue_classic_3_1_0.rb', 'db/migrate/update_queue_classic_3_1_0.rb'
+      end
     end
   end
 end

data/lib/generators/queue_classic/templates/update_queue_classic_3_1_0.rb

@@ -0,0 +1,9 @@
+class UpdateQueueClassic310 < ActiveRecord::Migration
+  def self.up
+    QC::Setup.update_to_3_1_0
+  end
+
+  def self.down
+    QC::Setup.downgrade_from_3_1_0
+  end
+end

data/lib/queue_classic.rb

@@ -57,7 +57,13 @@ module QC
   end
 
   def self.default_conn_adapter
-    @conn_adapter ||= ConnAdapter.new
+    return @conn_adapter if defined?(@conn_adapter) && @conn_adapter
+    if rails_connection_sharing_enabled?
+      @conn_adapter = ConnAdapter.new(ActiveRecord::Base.connection.raw_connection)
+    else
+      @conn_adapter = ConnAdapter.new
+    end
+    @conn_adapter
   end
 
   def self.default_conn_adapter=(conn)
@@ -81,7 +87,6 @@ module QC
     result = nil
     data = {:lib => "queue-classic"}.merge(data)
     if block_given?
-      start = Time.now
       result = yield
       data.merge(:elapsed => Integer((Time.now - t0)*1000))
     end
@@ -103,6 +108,13 @@ module QC
   def self.unlock_jobs_of_dead_workers
     @conn_adapter.execute("UPDATE #{QC::TABLE_NAME} SET locked_at = NULL, locked_by = NULL WHERE locked_by NOT IN (SELECT pid FROM pg_stat_activity);")
   end
+
+  private
+  def self.rails_connection_sharing_enabled?
+    enabled = ENV.fetch('QC_RAILS_DATABASE', 'true') != 'false'
+    return false unless enabled
+    return Object.const_defined?("ActiveRecord") && ActiveRecord::Base.respond_to?("connection")
+  end
 end
 
 require_relative "queue_classic/queue"

data/lib/queue_classic/conn_adapter.rb

@@ -64,8 +64,7 @@ module QC
 
     def validate!(c)
       return c if c.is_a?(PG::Connection)
-      klass = c.class
-      err = "connection must be an instance of PG::Connection, but was #{c}"
+      err = "connection must be an instance of PG::Connection, but was #{c.class}"
       raise(ArgumentError, err)
     end
 
@@ -94,7 +93,7 @@ module QC
     end
 
     def db_url
-      return @db_url if @db_url
+      return @db_url if defined?(@db_url) && @db_url
       url = ENV["QC_DATABASE_URL"] ||
             ENV["DATABASE_URL"]    ||
             raise(ArgumentError, "missing QC_DATABASE_URL or DATABASE_URL")

data/lib/queue_classic/queue.rb

@@ -36,7 +36,30 @@ module QC
     # `'hello', 'world'`.
     def enqueue(method, *args)
       QC.log_yield(:measure => 'queue.enqueue') do
-        s="INSERT INTO #{TABLE_NAME} (q_name, method, args) VALUES ($1, $2, $3)"
+        s = "INSERT INTO #{TABLE_NAME} (q_name, method, args) VALUES ($1, $2, $3)"
+        conn_adapter.execute(s, name, method, JSON.dump(args))
+      end
+    end
+
+    # enqueue_at(t,m,a) inserts a row into the jobs table representing a job
+    # to be executed not before the specified time.
+    # The time argument must be a Time object or a float timestamp. The method
+    # and args argument must be in the form described in the documentation for
+    # the #enqueue method.
+    def enqueue_at(timestamp, method, *args)
+      offset = Time.at(timestamp) - Time.now
+      enqueue_in(offset, method, *args)
+    end
+
+    # enqueue_in(t,m,a) inserts a row into the jobs table representing a job
+    # to be executed not before the specified time offset.
+    # The seconds argument must be an integer. The method and args argument
+    # must be in the form described in the documentation for the #enqueue
+    # method.
+    def enqueue_in(seconds, method, *args)
+      QC.log_yield(:measure => 'queue.enqueue') do
+        s = "INSERT INTO #{TABLE_NAME} (q_name, method, args, scheduled_at)
+             VALUES ($1, $2, $3, now() + interval '#{seconds.to_i} seconds')"
         res = conn_adapter.execute(s, name, method, JSON.dump(args))
       end
     end
@@ -50,9 +73,9 @@ module QC
             job[:q_name] = r["q_name"]
             job[:method] = r["method"]
             job[:args] = JSON.parse(r["args"])
-            if r["created_at"]
-              job[:created_at] = Time.parse(r["created_at"])
-              ttl = Integer((Time.now - job[:created_at]) * 1000)
+            if r["scheduled_at"]
+              job[:scheduled_at] = Time.parse(r["scheduled_at"])
+              ttl = Integer((Time.now - job[:scheduled_at]) * 1000)
               QC.measure("time-to-lock=#{ttl}ms source=#{name}")
             end
           end

data/lib/queue_classic/setup.rb

@@ -6,6 +6,8 @@ module QC
     DropSqlFunctions = File.join(Root, "/sql/drop_ddl.sql")
     UpgradeTo_3_0_0 = File.join(Root, "/sql/update_to_3_0_0.sql")
     DowngradeFrom_3_0_0 = File.join(Root, "/sql/downgrade_from_3_0_0.sql")
+    UpgradeTo_3_1_0 = File.join(Root, "/sql/update_to_3_1_0.sql")
+    DowngradeFrom_3_1_0 = File.join(Root, "/sql/downgrade_from_3_1_0.sql")
 
     def self.create(c = QC::default_conn_adapter.connection)
       conn = QC::ConnAdapter.new(c)
@@ -24,6 +26,7 @@ module QC
     def self.update(c = QC::default_conn_adapter.connection)
       conn = QC::ConnAdapter.new(c)
       conn.execute(File.read(UpgradeTo_3_0_0))
+      conn.execute(File.read(UpgradeTo_3_1_0))
       conn.execute(File.read(DropSqlFunctions))
       conn.execute(File.read(SqlFunctions))
     end
@@ -39,5 +42,17 @@ module QC
       conn = QC::ConnAdapter.new(c)
       conn.execute(File.read(DowngradeFrom_3_0_0))
     end
+
+    def self.update_to_3_1_0(c = QC::default_conn_adapter.connection)
+      conn = QC::ConnAdapter.new(c)
+      conn.execute(File.read(UpgradeTo_3_1_0))
+      conn.execute(File.read(DropSqlFunctions))
+      conn.execute(File.read(SqlFunctions))
+    end
+
+    def self.downgrade_from_3_1_0(c = QC::default_conn_adapter.connection)
+      conn = QC::ConnAdapter.new(c)
+      conn.execute(File.read(DowngradeFrom_3_1_0))
+    end
   end
 end

data/readme.md

@@ -1,15 +1,29 @@
 # queue_classic
 
-[![Build Status](https://travis-ci.org/QueueClassic/queue_classic.svg?branch=master)](https://travis-ci.org/QueueClassic/queue_classic)
-[![Code Climate](https://codeclimate.com/github/QueueClassic/queue_classic.png)](https://codeclimate.com/github/QueueClassic/queue_classic)
+<p align="center">
+  <b>Simple, efficient worker queue for Ruby & PostgreSQL</b>
+  <br />
+  <a href="https://travis-ci.org/QueueClassic/queue_classic"><img src="http://img.shields.io/travis/QueueClassic/queue_classic/master.svg?style=flat" /></a>
+
+  <a href="https://codeclimate.com/github/QueueClassic/queue_classic"><img src="http://img.shields.io/codeclimate/github/QueueClassic/queue_classic.svg?style=flat" /></a>
+
+  <a href="http://badge.fury.io/rb/queue_classic"><img src="http://img.shields.io/gem/v/queue_classic.svg?style=flat" alt="Gem Version" height="18"></a>
+</p>
+
+
+**IMPORTANT NOTE REGARDING VERSIONS**
+
+**This README is representing the current work for queue_classic 3.1, which is neither complete or stable. You can find the README for stable versions:**
+
+- latest stable can be found: [v3.0.X](https://github.com/QueueClassic/queue_classic/tree/3-0-stable)
+- older stable: [v2.2.3](https://github.com/QueueClassic/queue_classic/tree/v2.2.3)
 
-Current: v3.0.X
-Older stable: [v2.2.3](https://github.com/QueueClassic/queue_classic/tree/v2.2.3)
 
+## What is queue_classic?
 
 queue_classic provides a simple interface to a PostgreSQL-backed message queue. queue_classic specializes in concurrent locking and minimizing database load while providing a simple, intuitive developer experience. queue_classic assumes that you are already using PostgreSQL in your production environment and that adding another dependency (e.g. redis, beanstalkd, 0mq) is undesirable.
 
-Features:
+## Features
 
 * Leverage of PostgreSQL's listen/notify & row locking.
 * Support for multiple queues with heterogeneous workers.
@@ -18,12 +32,12 @@ Features:
 * Workers can work multiple queues.
 * Reduced row contention using a [relaxed FIFO](http://www.cs.tau.ac.il/~shanir/nir-pubs-web/Papers/Lock_Free.pdf) technique.
 
-Contents:
+## Table of content
 
 * [Documentation](http://rubydoc.info/gems/queue_classic/2.2.3/frames)
 * [Usage](#usage)
 * [Setup](#setup)
-* [Upgrade from V2 to V3](#upgrade-from-v2-to-v3)
+* [Upgrade from earlier versions to V3.1](#upgrade-from-earlier-versions)
 * [Configuration](#configuration)
   * [JSON](#json)
   * [Logging](#logging)
@@ -63,6 +77,16 @@ p_queue = QC::Queue.new("priority_queue")
 p_queue.enqueue("Kernel.puts", ["hello", "world"])
 ```
 
+There is also the possibility to schedule a job at a specified time in the future. It will not be worked off before that specified time.
+
+```ruby
+# Specifying the job execution time exactly.
+QC.enqueue_at(Time.new(2024,01,02,10,00), "Kernel.puts", "hello future")
+
+# Specifying the job execution time as an offset in seconds.
+QC.enqueue_in(60, "Kernel.puts", "hello from 1 minute later")
+```
+
 ### Working Jobs
 
 There are two ways to work jobs. The first approach is to use the Rake task. The second approach is to use a custom executable.
@@ -88,7 +112,7 @@ $ QUEUE="priority_queue" bundle exec rake qc:work
 
 Setup a worker to work multiple queues.
 ```bash
-$ QUEUES="priority_queue, secondary_queue" bundle exec rake qc:work
+$ QUEUES="priority_queue,secondary_queue" bundle exec rake qc:work
 ```
 In this scenario, on each iteration of the worker's loop, it will look for jobs in the first queue prior to looking at the second queue. This means that the first queue must be empty before the worker will look at the second queue.
 
@@ -148,17 +172,7 @@ rails generate queue_classic:install
 rake db:migrate
 ```
 
-By default, queue_classic will use the QC_DATABASE_URL falling back on DATABASE_URL. The URL must be in the following format: `postgres://username:password@localhost/database_name`.  If you use Heroku's PostgreSQL service, this will already be set. If you don't want to set this variable, you can set the connection in an initializer. **QueueClassic will maintain its own connection to the database.** This may double the number of connections to your database.
-
-You can share your active record connection with queue_classic –**however this is not thread safe.**
-
-```ruby
-require 'queue_classic'
-QC.default_conn_adapter = QC::ConnAdapter.new(
-	ActiveRecord::Base.connection.raw_connection)
-```
-
-**Note on using ActiveRecord migrations:** If you use the migration, and you wish to use commands that reset the database from the stored schema (e.g. `rake db:reset`), your application must be configured with `config.active_record.schema_format = :sql` in `config/application.rb`.  If you don't do this, the PL/pgSQL function that queue_classic creates will be lost when you reset the database.
+If you want to use queue_classic with Active Job (Rails 4.2+), you need to set `Rails.application.config.active_job.queue_adapter = :queue_classic`. Everything else will be taken care for you, just use the Active Job API.
 
 ### Rake Task Setup
 
@@ -172,7 +186,22 @@ $ bundle exec rake qc:create
 $ bundle exec rake qc:drop
 ```
 
-## Upgrade from V2 to V3
+### Database connection
+
+#### Ruby on Rails
+
+Starting with with queue_classic 3.1, Rails is automatically detected and its connection is used.
+
+If you don't want to use the automatic database connection, set this environment variable to false: `export QC_RAILS_DATABASE=false`
+
+**Note on using ActiveRecord migrations:** If you use the migration, and you wish to use commands that reset the database from the stored schema (e.g. `rake db:reset`), your application must be configured with `config.active_record.schema_format = :sql` in `config/application.rb`.  If you don't do this, the PL/pgSQL function that queue_classic creates will be lost when you reset the database.
+
+
+#### Other Ruby apps
+
+By default, queue_classic will use the QC_DATABASE_URL falling back on DATABASE_URL. The URL must be in the following format: `postgres://username:password@localhost/database_name`.  If you use Heroku's PostgreSQL service, this will already be set. If you don't want to set this variable, you can set the connection in an initializer. **QueueClassic will maintain its own connection to the database.** This may double the number of connections to your database.
+
+## Upgrade from earlier versions
 If you are upgrading from a previous version of queue_classic, you might need some new database columns and/or functions. Luckily enough for you, it is easy to do so.
 
 ### Ruby on Rails

data/sql/create_table.sql

@@ -7,7 +7,8 @@ CREATE TABLE queue_classic_jobs (
   args   text not null,
   locked_at timestamptz,
   locked_by integer,
-  created_at timestamptz default now()
+  created_at timestamptz default now(),
+  scheduled_at timestamptz default now()
 );
 
 -- If json type is available, use it for the args column.
@@ -19,3 +20,5 @@ end if;
 end $$ language plpgsql;
 
 CREATE INDEX idx_qc_on_name_only_unlocked ON queue_classic_jobs (q_name, id) WHERE locked_at IS NULL;
+CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;
+

data/sql/ddl.sql

@@ -19,6 +19,8 @@ BEGIN
     || ' WHERE locked_at IS NULL'
     || ' AND q_name = '
     || quote_literal(q_name)
+    || ' AND scheduled_at <= '
+    || quote_literal(now())
     || ' LIMIT '
     || quote_literal(top_boundary)
     || ') limited'
@@ -37,6 +39,8 @@ BEGIN
         || ' WHERE locked_at IS NULL'
         || ' AND q_name = '
         || quote_literal(q_name)
+        || ' AND scheduled_at <= '
+        || quote_literal(now())
         || ' ORDER BY id ASC'
         || ' LIMIT 1'
         || ' OFFSET ' || quote_literal(relative_top)

data/sql/downgrade_from_3_1_0.sql

@@ -0,0 +1 @@
+ALTER TABLE queue_classic_jobs DROP COLUMN scheduled_at;

data/sql/update_to_3_1_0.sql

@@ -0,0 +1,9 @@
+DO $$DECLARE r record;
+BEGIN
+	BEGIN
+		ALTER TABLE queue_classic_jobs ADD COLUMN scheduled_at timestamptz default now();
+		CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;
+	EXCEPTION
+		WHEN duplicate_column THEN RAISE NOTICE 'column scheduled_at already exists in queue_classic_jobs.';
+	END;
+END$$;

data/test/lib/queue_classic_rails_connection_test.rb

@@ -0,0 +1,40 @@
+require File.expand_path("../../helper.rb", __FILE__)
+
+class QueueClassicRailsConnectionTest < QCTest
+  def before_setup
+    Object.send :const_set, :ActiveRecord, Module.new
+    ActiveRecord.const_set :Base, Module.new
+
+    QC.default_conn_adapter = nil
+  end
+
+  def after_teardown
+    ActiveRecord.send :remove_const, :Base
+    Object.send :remove_const, :ActiveRecord
+  end
+
+  def test_uses_active_record_connection_if_exists
+    connection = get_connection
+    assert connection.verify
+  end
+
+  def test_does_not_use_active_record_connection_if_env_var_set
+    ENV['QC_RAILS_DATABASE'] = 'false'
+    connection = get_connection
+    assert_raises(MockExpectationError) { connection.verify }
+    ENV['QC_RAILS_DATABASE'] = 'true'
+  end
+
+  private
+  def get_connection
+    connection = Minitest::Mock.new
+    connection.expect(:raw_connection, QC::ConnAdapter.new.connection)
+
+    ActiveRecord::Base.define_singleton_method(:connection) do
+      connection
+    end
+
+    QC.default_conn_adapter
+    connection
+  end
+end

data/test/queue_test.rb

@@ -27,6 +27,35 @@ class QueueTest < QCTest
     assert_nil(QC.lock)
   end
 
+  def test_lock_with_future_job_with_enqueue_in
+    QC.enqueue_in(2, "Klass.method")
+    assert_nil QC.lock
+    sleep 2
+    job = QC.lock
+    assert_equal("Klass.method", job[:method])
+    assert_equal([], job[:args])
+  end
+
+  def test_lock_with_future_job_with_enqueue_at_with_a_time_object
+    future = Time.now + 2
+    QC.enqueue_at(future, "Klass.method")
+    assert_nil QC.lock
+    until Time.now >= future do sleep 0.1 end
+    job = QC.lock
+    assert_equal("Klass.method", job[:method])
+    assert_equal([], job[:args])
+  end
+
+  def test_lock_with_future_job_with_enqueue_at_with_a_float_timestamp
+    offset = (Time.now + 2).to_f
+    QC.enqueue_at(offset, "Klass.method")
+    assert_nil QC.lock
+    sleep 2
+    job = QC.lock
+    assert_equal("Klass.method", job[:method])
+    assert_equal([], job[:args])
+  end
+
   def test_count
     QC.enqueue("Klass.method")
     assert_equal(1, QC.count)

data/test/worker_test.rb

@@ -179,7 +179,7 @@ class WorkerTest < QCTest
     set_database 'postgres:///invalid'
 
     conn = PG::Connection.connect(dbname: 'queue_classic_test')
-    worker = QC::Worker.new connection: conn
+    QC::Worker.new connection: conn
 
     conn.close
     reset_database

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: queue_classic
 version: !ruby/object:Gem::Version
-  version: 3.0.3
+  version: 3.1.0.RC1
 platform: ruby
 authors:
 - Ryan Smith (♠ ace hacker)
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-05 00:00:00.000000000 Z
+date: 2015-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
@@ -42,6 +42,7 @@ files:
 - lib/generators/queue_classic/templates/add_queue_classic.rb
 - lib/generators/queue_classic/templates/update_queue_classic_3_0_0.rb
 - lib/generators/queue_classic/templates/update_queue_classic_3_0_2.rb
+- lib/generators/queue_classic/templates/update_queue_classic_3_1_0.rb
 - lib/queue_classic.rb
 - lib/queue_classic/conn_adapter.rb
 - lib/queue_classic/queue.rb
@@ -53,10 +54,13 @@ files:
 - sql/create_table.sql
 - sql/ddl.sql
 - sql/downgrade_from_3_0_0.sql
+- sql/downgrade_from_3_1_0.sql
 - sql/drop_ddl.sql
 - sql/update_to_3_0_0.sql
+- sql/update_to_3_1_0.sql
 - test/benchmark_test.rb
 - test/helper.rb
+- test/lib/queue_classic_rails_connection_test.rb
 - test/lib/queue_classic_test.rb
 - test/queue_test.rb
 - test/worker_test.rb
@@ -75,9 +79,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.2.2
@@ -86,6 +90,7 @@ specification_version: 4
 summary: Simple, efficient worker queue for Ruby & PostgreSQL.
 test_files:
 - test/benchmark_test.rb
+- test/lib/queue_classic_rails_connection_test.rb
 - test/lib/queue_classic_test.rb
 - test/queue_test.rb
 - test/worker_test.rb