que 0.6.0 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f698e236aff233a785bc44075257a17c1ae35e3b
-  data.tar.gz: 0916c6cbb864a9c148bb43e9765cd569330abdf9
+  metadata.gz: a18d2e13eaf6870dbd5e912bf072b54d4f45e79d
+  data.tar.gz: 74a6bef98e5c1911dcf944c05392941445f71da5
 SHA512:
-  metadata.gz: 4bad7602663617ced6acf5e76cd041c8f6ec7ed0339b7706ca2121e305cb61bb4a55b2bdc1752c7f5f14ad2147e6e65df7e6d1a0192ca5bb25d12dd02f123b90
-  data.tar.gz: e7719dc683f696209c4f5dcdb07495693bb891d2a6dbace5ec7be9954037234024b0d246c8394a62f75272dc03acdcbbfea43f517acd9669467a352672a410bb
+  metadata.gz: 22d13ee07b45cab14849d810ffa714d9f773b13a3509f9e5c20e10b1e192c760c4903eed5d8d1721614a29d85b4542d51c3aa440d491047b5cb540e0c497122b
+  data.tar.gz: 8c8cab41047a14bf177e8394439213f4aef0e4688e53f9d99d765ce12d0bd5489db8bd36f335e2e76bf72a19fba4e0f6f3c22670650d3786ff8a6f939ba849dc

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+### 0.7.0 (2014-04-09)
+
+*   `JobClass.queue(*args)` has been deprecated and will be removed in version 1.0.0. Please use `JobClass.enqueue(*args)` instead.
+
+*   The `@default_priority` and `@default_run_at` variables have been deprecated and will be removed in version 1.0.0. Please use `@priority` and `@run_at` instead, respectively.
+
+*   Log lines now include the process pid - its omission in the previous release was an oversight.
+
+*   The [Pond gem](https://github.com/chanks/pond) is now supported as a connection. It is very similar to the ConnectionPool gem, but creates connections lazily and is dynamically resizable.
+
 ### 0.6.0 (2014-02-04)
 
 *   **A schema upgrade to version 3 is required for this release.** See [the migration doc](https://github.com/chanks/que/blob/master/docs/migrating.md) for information if you're upgrading from a previous release.

data/Gemfile

@@ -6,6 +6,7 @@ group :development, :test do
   gem 'activerecord',    :require => nil
   gem 'sequel',          :require => nil
   gem 'connection_pool', :require => nil
+  gem 'pond',            :require => nil
   gem 'pg',              :require => nil, :platform => :ruby
   gem 'pg_jruby',        :require => nil, :platform => :jruby
 end

data/docs/managing_workers.md

@@ -25,6 +25,10 @@ If you don't want to burden your web processes with too much work and want to ru
     # Or configure the number of workers:
     QUE_WORKER_COUNT=8 rake que:work
 
+Other options available via environment variables are `QUE_QUEUE` to determine which named queue jobs are pulled from, and `QUE_WAKE_INTERVAL` to determine how long workers will wait to poll again when there are no jobs available. For example, to run 2 workers that run jobs from the "other_queue" queue and wait a half-second between polls, you could do:
+
+    QUE_QUEUE=other_queue QUE_WORKER_COUNT=2 QUE_WAKE_INTERVAL=0.5 rake que:work
+
 ### Thread-Unsafe Application Code
 
 If your application code is not thread-safe, you won't want any workers to be processing jobs while anything else is happening in the Ruby process. So, you'll want to turn the worker pool off by default:

data/docs/multiple_queues.md

@@ -8,6 +8,14 @@ Then you can set jobs to be enqueued in that queue specifically:
 
     ProcessCreditCard.enqueue current_user.id, :queue => 'credit_cards'
 
+    # Or:
+
+    class ProcessCreditCard < Que::Job
+      # Set a default queue for this job class; this can be overridden by
+      # passing the :queue parameter to enqueue like above.
+      @queue = 'credit_cards'
+    end
+
 In some cases, the ProcessCreditCard class may not be defined in the application that is enqueueing the job. In that case, you can specify the job class as a string:
 
     Que.enqueue current_user.id, :job_class => 'ProcessCreditCard', :queue => 'credit_cards'

data/docs/using_plain_connections.md

@@ -31,4 +31,20 @@ If you want to be able to use multithreading to run multiple jobs simultaneously
 
 Be sure to pick your pool size carefully - if you use 10 for the size, you'll incur the overhead of having 10 connections open to Postgres even if you never use more than a couple of them.
 
-Please be aware that if you're using ActiveRecord or Sequel to manage your data, there's no reason for you to be using either of these methods - it's less efficient (unnecessary connections will waste memory on your database server) and you lose the reliability benefits of wrapping jobs in the same transactions as the rest of your data.
+The Pond gem doesn't have this drawback - it is very similar to ConnectionPool, but establishes connections lazily (add `gem 'pond'` to your Gemfile):
+
+    require 'uri'
+    require 'pg'
+    require 'pond'
+
+    uri = URI.parse(ENV['DATABASE_URL'])
+
+    Que.connection = Pond.new :maximum_size => 10 do
+      PG::Connection.open :host     => uri.host,
+                          :user     => uri.user,
+                          :password => uri.password,
+                          :port     => uri.port || 5432,
+                          :dbname   => uri.path[1..-1]
+    end
+
+Please be aware that if you're using ActiveRecord or Sequel to manage your data, there's no reason for you to be using any of these methods - it's less efficient (unnecessary connections will waste memory on your database server) and you lose the reliability benefits of wrapping jobs in the same transactions as the rest of your data.

data/lib/que.rb

@@ -21,13 +21,15 @@ module Que
     attr_writer :adapter, :log_formatter
 
     def connection=(connection)
-      self.adapter = if connection.to_s == 'ActiveRecord'
-        Adapters::ActiveRecord.new
-      else
-        case connection.class.to_s
+      self.adapter =
+        if connection.to_s == 'ActiveRecord'
+          Adapters::ActiveRecord.new
+        else
+          case connection.class.to_s
           when 'Sequel::Postgres::Database' then Adapters::Sequel.new(connection)
           when 'ConnectionPool'             then Adapters::ConnectionPool.new(connection)
           when 'PG::Connection'             then Adapters::PG.new(connection)
+          when 'Pond'                       then Adapters::Pond.new(connection)
           when 'NilClass'                   then connection
           else raise "Que connection not recognized: #{connection.inspect}"
         end
@@ -79,7 +81,7 @@ module Que
 
     def log(data)
       level = data.delete(:level) || :info
-      data = {:lib => 'que', :hostname => Socket.gethostname, :thread => Thread.current.object_id}.merge(data)
+      data = {:lib => 'que', :hostname => Socket.gethostname, :pid => Process.pid, :thread => Thread.current.object_id}.merge(data)
 
       if logger && output = log_formatter.call(data)
         logger.send level, output

data/lib/que/adapters/base.rb

@@ -5,6 +5,7 @@ module Que
     autoload :ActiveRecord,   'que/adapters/active_record'
     autoload :ConnectionPool, 'que/adapters/connection_pool'
     autoload :PG,             'que/adapters/pg'
+    autoload :Pond,           'que/adapters/pond'
     autoload :Sequel,         'que/adapters/sequel'
 
     class Base

data/lib/que/adapters/pond.rb

@@ -0,0 +1,14 @@
+module Que
+  module Adapters
+    class Pond < Base
+      def initialize(pond)
+        @pond = pond
+        super
+      end
+
+      def checkout(&block)
+        @pond.checkout(&block)
+      end
+    end
+  end
+end

data/lib/que/job.rb

@@ -40,10 +40,14 @@ module Que
 
         attrs = {:job_class => job_class || to_s, :args => args}
 
+        warn "@default_run_at in #{to_s} has been deprecated and will be removed in Que version 1.0.0. Please use @run_at instead." if @default_run_at
+
         if t = run_at || @run_at && @run_at.call || @default_run_at && @default_run_at.call
           attrs[:run_at] = t
         end
 
+        warn "@default_priority in #{to_s} has been deprecated and will be removed in Que version 1.0.0. Please use @priority instead." if @default_priority
+
         if p = priority || @priority || @default_priority
           attrs[:priority] = p
         end
@@ -61,7 +65,10 @@ module Que
         end
       end
 
-      alias queue enqueue
+      def queue(*args)
+        warn "#{to_s}.queue(*args) is deprecated and will be removed in Que version 1.0.0. Please use #{to_s}.enqueue(*args) instead."
+        enqueue(*args)
+      end
 
       def run(*args)
         new(:args => args).tap(&:_run)
@@ -98,7 +105,7 @@ module Que
             begin
               if job
                 count    = job[:error_count].to_i + 1
-                interval = klass && klass.retry_interval || retry_interval
+                interval = klass && klass.respond_to?(:retry_interval) && klass.retry_interval || retry_interval
                 delay    = interval.respond_to?(:call) ? interval.call(count) : interval
                 message  = "#{error.message}\n#{error.backtrace.join("\n")}"
                 Que.execute :set_error, [count, delay, message] + job.values_at(:queue, :priority, :run_at, :job_id)

data/lib/que/rake_tasks.rb

@@ -12,7 +12,9 @@ namespace :que do
     # the rake task in tasks/safe_shutdown.rb.
 
     stop = false
-    trap('INT'){stop = true}
+    %w( INT TERM ).each do |signal|
+      trap(signal) {stop = true}
+    end
 
     at_exit do
       $stdout.puts "Finishing Que's current jobs before exiting..."
@@ -26,9 +28,9 @@ namespace :que do
     end
   end
 
-  desc "Create Que's job table"
-  task :create => :environment do
-    Que.create!
+  desc "Migrate Que's job table to the most recent version (creating it if it doesn't exist)"
+  task :migrate => :environment do
+    Que.migrate!
   end
 
   desc "Drop Que's job table"

data/lib/que/version.rb

@@ -1,3 +1,3 @@
 module Que
-  Version = '0.6.0'
+  Version = '0.7.0'
 end

data/lib/que/worker.rb

@@ -15,6 +15,7 @@ module Que
       @queue  = queue
       @state  = :working
       @thread = Thread.new { work_loop }
+      @thread.abort_on_exception = true
     end
 
     def alive?

data/spec/adapters/pond_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+require 'pond'
+
+Que.connection = QUE_SPEC_POND = Pond.new &NEW_PG_CONNECTION
+QUE_ADAPTERS[:pond] = Que.adapter
+
+describe "Que using the Pond adapter" do
+  before { Que.adapter = QUE_ADAPTERS[:pond] }
+
+  it_behaves_like "a multi-threaded Que adapter"
+
+  it "should be able to tell when it's already in a transaction" do
+    Que.adapter.should_not be_in_transaction
+    QUE_SPEC_POND.checkout do |conn|
+      conn.async_exec "BEGIN"
+      Que.adapter.should be_in_transaction
+      conn.async_exec "COMMIT"
+    end
+  end
+end

data/spec/support/helpers.rb

@@ -8,3 +8,10 @@ def sleep_until(timeout = QUE_TEST_TIMEOUT)
     sleep 0.01
   end
 end
+
+def suppress_warnings
+  original_verbosity, $VERBOSE = $VERBOSE, nil
+  yield
+ensure
+  $VERBOSE = original_verbosity
+end

data/spec/unit/customization_spec.rb

@@ -34,7 +34,7 @@ describe "Customizing Que" do
         end
 
         def method_missing(method, *args)
-          Delayed.queue Marshal.dump(@receiver), method, Marshal.dump(args)
+          Delayed.enqueue Marshal.dump(@receiver), method, Marshal.dump(args)
         end
       end
 

data/spec/unit/enqueue_spec.rb

@@ -21,7 +21,7 @@ describe Que::Job, '.enqueue' do
 
   it "should be aliased to .queue" do
     DB[:que_jobs].count.should be 0
-    Que::Job.queue
+    suppress_warnings { Que::Job.queue }
     DB[:que_jobs].count.should be 1
   end
 
@@ -146,8 +146,10 @@ describe Que::Job, '.enqueue' do
     end
 
     DB[:que_jobs].count.should be 0
-    OldDefaultPriorityJob.enqueue 1
-    OldDefaultPriorityJob.enqueue 1, :priority => 4
+    suppress_warnings do
+      OldDefaultPriorityJob.enqueue 1
+      OldDefaultPriorityJob.enqueue 1, :priority => 4
+    end
     DB[:que_jobs].count.should be 2
 
     first, second = DB[:que_jobs].order(:job_id).all
@@ -196,8 +198,10 @@ describe Que::Job, '.enqueue' do
     end
 
     DB[:que_jobs].count.should be 0
-    OldDefaultRunAtJob.enqueue 1
-    OldDefaultRunAtJob.enqueue 1, :run_at => Time.now + 30
+    suppress_warnings do
+      OldDefaultRunAtJob.enqueue 1
+      OldDefaultRunAtJob.enqueue 1, :run_at => Time.now + 30
+    end
     DB[:que_jobs].count.should be 2
 
     first, second = DB[:que_jobs].order(:job_id).all

data/spec/unit/logging_spec.rb

@@ -8,6 +8,7 @@ describe "Logging" do
     message = JSON.load($logger.messages.first)
     message['lib'].should == 'que'
     message['hostname'].should == Socket.gethostname
+    message['pid'].should == Process.pid
     message['event'].should == 'blah'
     message['source'].should == 4
     message['thread'].should == Thread.current.object_id

data/spec/unit/work_spec.rb

@@ -349,5 +349,23 @@ describe Que::Job, '.work' do
       job[:last_error].should =~ /uninitialized constant:? NonexistentClass/
       job[:run_at].should be_within(3).of Time.now + 4
     end
+
+    it "should throw an error properly if the corresponding job class doesn't descend from Que::Job" do
+      class J
+        def run(*args)
+        end
+      end
+
+      Que.enqueue :job_class => "J"
+
+      result = Que::Job.work
+      result[:event].should == :job_errored
+      result[:job][:job_class].should == 'J'
+
+      DB[:que_jobs].count.should be 1
+      job = DB[:que_jobs].first
+      job[:error_count].should be 1
+      job[:run_at].should be_within(3).of Time.now + 4
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: que
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Chris Hanks
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-05 00:00:00.000000000 Z
+date: 2014-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -58,6 +58,7 @@ files:
 - lib/que/adapters/base.rb
 - lib/que/adapters/connection_pool.rb
 - lib/que/adapters/pg.rb
+- lib/que/adapters/pond.rb
 - lib/que/adapters/sequel.rb
 - lib/que/job.rb
 - lib/que/migrations.rb
@@ -76,6 +77,7 @@ files:
 - spec/adapters/active_record_spec.rb
 - spec/adapters/connection_pool_spec.rb
 - spec/adapters/pg_spec.rb
+- spec/adapters/pond_spec.rb
 - spec/adapters/sequel_spec.rb
 - spec/gemfiles/Gemfile1
 - spec/gemfiles/Gemfile2
@@ -128,6 +130,7 @@ test_files:
 - spec/adapters/active_record_spec.rb
 - spec/adapters/connection_pool_spec.rb
 - spec/adapters/pg_spec.rb
+- spec/adapters/pond_spec.rb
 - spec/adapters/sequel_spec.rb
 - spec/gemfiles/Gemfile1
 - spec/gemfiles/Gemfile2