que 0.4.0 → 0.5.0

data/que.gemspec

@@ -19,6 +19,4 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_development_dependency 'bundler', '~> 1.3'
-
-  spec.add_dependency 'multi_json', '~> 1.0'
 end

data/spec/adapters/active_record_spec.rb

@@ -14,17 +14,21 @@ unless defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
     it_behaves_like "a multi-threaded Que adapter"
 
     it "should use the same connection that ActiveRecord does" do
-      class ActiveRecordJob < Que::Job
-        def run
-          $pid1 = Que.execute("SELECT pg_backend_pid()").first['pg_backend_pid'].to_i
-          $pid2 = ActiveRecord::Base.connection.select_all("select pg_backend_pid()").rows.first.first.to_i
+      begin
+        class ActiveRecordJob < Que::Job
+          def run
+            $pid1 = Integer(Que.execute("select pg_backend_pid()").first['pg_backend_pid'])
+            $pid2 = Integer(ActiveRecord::Base.connection.select_value("select pg_backend_pid()"))
+          end
         end
-      end
 
-      ActiveRecordJob.queue
-      Que::Job.work
+        ActiveRecordJob.queue
+        Que::Job.work
 
-      $pid1.should == $pid2
+        $pid1.should == $pid2
+      ensure
+        $pid1 = $pid2 = nil
+      end
     end
 
     it "should instantiate args as ActiveSupport::HashWithIndifferentAccess" do
@@ -44,5 +48,31 @@ unless defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby'
       Que.wake_interval = 0.005.seconds
       sleep_until { DB[:que_jobs].empty? }
     end
+
+    it "should wake up a Worker after queueing a job in async mode, waiting for a transaction to commit if necessary" do
+      Que.mode = :async
+      sleep_until { Que::Worker.workers.all? &:sleeping? }
+
+      # Wakes a worker immediately when not in a transaction.
+      Que::Job.queue
+      sleep_until { Que::Worker.workers.all?(&:sleeping?) && DB[:que_jobs].empty? }
+
+      ActiveRecord::Base.transaction do
+        Que::Job.queue
+        Que::Worker.workers.each { |worker| worker.should be_sleeping }
+      end
+      sleep_until { Que::Worker.workers.all?(&:sleeping?) && DB[:que_jobs].empty? }
+
+      # Do nothing when queueing with a specific :run_at.
+      BlockJob.queue :run_at => Time.now
+      Que::Worker.workers.each { |worker| worker.should be_sleeping }
+    end
+
+    it "should be able to tell when it's in an ActiveRecord transaction" do
+      Que.adapter.should_not be_in_transaction
+      ActiveRecord::Base.transaction do
+        Que.adapter.should be_in_transaction
+      end
+    end
   end
 end

data/spec/adapters/connection_pool_spec.rb

@@ -1,11 +1,20 @@
 require 'spec_helper'
 require 'connection_pool'
 
-Que.connection = ConnectionPool.new &NEW_PG_CONNECTION
+Que.connection = QUE_SPEC_CONNECTION_POOL = ConnectionPool.new &NEW_PG_CONNECTION
 QUE_ADAPTERS[:connection_pool] = Que.adapter
 
 describe "Que using the ConnectionPool adapter" do
   before { Que.adapter = QUE_ADAPTERS[:connection_pool] }
 
   it_behaves_like "a multi-threaded Que adapter"
+
+  it "should be able to tell when it's already in a transaction" do
+    Que.adapter.should_not be_in_transaction
+    QUE_SPEC_CONNECTION_POOL.with do |conn|
+      conn.async_exec "BEGIN"
+      Que.adapter.should be_in_transaction
+      conn.async_exec "COMMIT"
+    end
+  end
 end

data/spec/adapters/sequel_spec.rb

@@ -9,17 +9,21 @@ describe "Que using the Sequel adapter" do
   it_behaves_like "a multi-threaded Que adapter"
 
   it "should use the same connection that Sequel does" do
-    class SequelJob < Que::Job
-      def run
-        $pid1 = Que.execute("SELECT pg_backend_pid()").first['pg_backend_pid'].to_i
-        $pid2 = SEQUEL_ADAPTER_DB.get{pg_backend_pid{}}
+    begin
+      class SequelJob < Que::Job
+        def run
+          $pid1 = Integer(Que.execute("select pg_backend_pid()").first['pg_backend_pid'])
+          $pid2 = Integer(SEQUEL_ADAPTER_DB['select pg_backend_pid()'].get)
+        end
       end
-    end
 
-    SequelJob.queue
-    Que::Job.work
+      SequelJob.queue
+      Que::Job.work
 
-    $pid1.should == $pid2
+      $pid1.should == $pid2
+    ensure
+      $pid1 = $pid2 = nil
+    end
   end
 
   it "should wake up a Worker after queueing a job in async mode, waiting for a transaction to commit if necessary" do
@@ -40,4 +44,11 @@ describe "Que using the Sequel adapter" do
     BlockJob.queue :run_at => Time.now
     Que::Worker.workers.each { |worker| worker.should be_sleeping }
   end
+
+  it "should be able to tell when it's in a Sequel transaction" do
+    Que.adapter.should_not be_in_transaction
+    SEQUEL_ADAPTER_DB.transaction do
+      Que.adapter.should be_in_transaction
+    end
+  end
 end

data/spec/spec_helper.rb

@@ -1,8 +1,8 @@
 require 'que'
 require 'uri'
 require 'pg'
-require 'json'
 require 'logger'
+require 'json'
 
 Dir['./spec/support/**/*.rb'].sort.each &method(:require)
 
@@ -42,8 +42,12 @@ QUE_ADAPTERS = {:pg => Que.adapter}
 # We use Sequel to examine the database in specs.
 require 'sequel'
 DB = Sequel.connect(QUE_URL)
+
+
+
+# Reset the table to the most up-to-date version.
 DB.drop_table? :que_jobs
-DB.run Que::SQL[:create_table]
+Que::Migrations.migrate!
 
 
 

data/spec/support/shared_examples/adapter.rb

@@ -7,7 +7,9 @@ shared_examples "a Que adapter" do
 
   it "should be able to queue and work a job" do
     Que::Job.queue
-    Que::Job.work.should be_an_instance_of Que::Job
+    result = Que::Job.work
+    result[:event].should == :job_worked
+    result[:job][:job_class].should == 'Que::Job'
   end
 
   it "should yield the same Postgres connection for the duration of the block" do

data/spec/travis.rb

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+# Run tests a bunch of times, flush out thread race conditions /  errors.
+test_runs = if ENV['TESTS']
+              Integer(ENV['TESTS'])
+            else
+              50
+            end
+
+1.upto(test_runs) do |i|
+  puts "Test Run #{i}"
+  exit(-1) if !system("bundle exec rake")
+end

data/spec/unit/helper_spec.rb

@@ -1,14 +1,6 @@
 require 'spec_helper'
 
 describe Que, 'helpers' do
-  it "should be able to drop and create the jobs table" do
-    DB.table_exists?(:que_jobs).should be true
-    Que.drop!
-    DB.table_exists?(:que_jobs).should be false
-    Que.create!
-    DB.table_exists?(:que_jobs).should be true
-  end
-
   it "should be able to clear the jobs table" do
     DB[:que_jobs].insert :job_class => "Que::Job"
     DB[:que_jobs].count.should be 1

data/spec/unit/logging_spec.rb

@@ -0,0 +1,75 @@
+require 'spec_helper'
+
+describe "Logging" do
+  it "by default should record the library and thread id in JSON" do
+    Que.log :event => "blah", :source => 4
+    $logger.messages.count.should be 1
+
+    message = JSON.load($logger.messages.first)
+    message['lib'].should == 'que'
+    message['event'].should == 'blah'
+    message['source'].should == 4
+    message['thread'].should == Thread.current.object_id
+  end
+
+  it "should not raise an error when no logger is present" do
+    begin
+      Que.logger = nil
+
+      Que::Job.queue
+      worker = Que::Worker.new
+      sleep_until { worker.sleeping? }
+
+      DB[:que_jobs].should be_empty
+
+      worker.stop
+      worker.wait_until_stopped
+    ensure
+      Que.logger = $logger
+    end
+  end
+
+  it "should allow the use of a custom log formatter" do
+    begin
+      Que.log_formatter = proc { |data| "Logged event is #{data[:event]}" }
+      Que.log :event => 'my_event'
+      $logger.messages.count.should be 1
+      $logger.messages.first.should == "Logged event is my_event"
+    ensure
+      Que.log_formatter = nil
+    end
+  end
+
+  it "should not log anything if the logging formatter returns falsey" do
+    begin
+      Que.log_formatter = proc { |data| false }
+
+      Que.log :event => "blah"
+      $logger.messages.should be_empty
+    ensure
+      Que.log_formatter = nil
+    end
+  end
+
+  it "should use a :level option to set the log level if one exists, or default to info" do
+    begin
+      Que.logger = o = Object.new
+
+      def o.method_missing(level, message)
+        $level = level
+        $message = message
+      end
+
+      Que.log :message => 'one'
+      $level.should == :info
+      JSON.load($message)['message'].should == 'one'
+
+      Que.log :message => 'two', :level => 'debug'
+      $level.should == :debug
+      JSON.load($message)['message'].should == 'two'
+    ensure
+      Que.logger = $logger
+      $level = $message = nil
+    end
+  end
+end

data/spec/unit/migrations_spec.rb

@@ -0,0 +1,114 @@
+require 'spec_helper'
+
+describe Que::Migrations do
+  it "should be able to perform migrations up and down" do
+    # Migration #1 creates the table with a priority default of 1, migration
+    # #2 ups that to 100.
+
+    default = proc do
+      result = Que.execute <<-SQL
+        select adsrc
+        from pg_attribute a
+        join pg_class c on c.oid = a.attrelid
+        join pg_attrdef on adrelid = attrelid AND adnum = attnum
+        where relname = 'que_jobs'
+        and attname = 'priority'
+      SQL
+
+      result.first['adsrc'].to_i
+    end
+
+    default.call.should == 100
+    Que::Migrations.migrate! :version => 1
+    default.call.should == 1
+    Que::Migrations.migrate! :version => 2
+    default.call.should == 100
+
+    # Clean up.
+    Que.migrate!
+  end
+
+  it "should be able to get and set the current schema version" do
+    Que::Migrations.db_version.should == Que::Migrations::CURRENT_VERSION
+    Que::Migrations.set_db_version(59328)
+    Que::Migrations.db_version.should == 59328
+    Que::Migrations.set_db_version(Que::Migrations::CURRENT_VERSION)
+    Que::Migrations.db_version.should == Que::Migrations::CURRENT_VERSION
+  end
+
+  it "should be able to cycle the jobs table all the way between nonexistent and current without error" do
+    Que::Migrations.db_version.should == Que::Migrations::CURRENT_VERSION
+    Que::Migrations.migrate! :version => 0
+    Que::Migrations.db_version.should == 0
+    Que::Migrations.migrate!
+    Que::Migrations.db_version.should == Que::Migrations::CURRENT_VERSION
+
+    # The helper on the Que module does the same thing.
+    Que.migrate! :version => 0
+    Que::Migrations.db_version.should == 0
+    Que.migrate!
+    Que::Migrations.db_version.should == Que::Migrations::CURRENT_VERSION
+  end
+
+  it "should be able to honor the initial behavior of Que.drop!" do
+    DB.table_exists?(:que_jobs).should be true
+    Que.drop!
+    DB.table_exists?(:que_jobs).should be false
+
+    # Clean up.
+    Que::Migrations.migrate!
+    DB.table_exists?(:que_jobs).should be true
+  end
+
+  it "should be able to recognize a que_jobs table created before the versioning system" do
+    DB.drop_table :que_jobs
+    DB.create_table(:que_jobs){serial :id} # Dummy Table.
+    Que::Migrations.db_version.should == 1
+    DB.drop_table(:que_jobs)
+    Que::Migrations.migrate!
+  end
+
+  it "should be able to honor the initial behavior of Que.create!" do
+    DB.drop_table :que_jobs
+    Que.create!
+    DB.table_exists?(:que_jobs).should be true
+    Que::Migrations.db_version.should == 1
+
+    # Clean up.
+    Que::Migrations.migrate!
+    DB.table_exists?(:que_jobs).should be true
+  end
+
+  it "should use transactions to protect its migrations from errors" do
+    proc do
+      Que::Migrations.transaction do
+        Que.execute "DROP TABLE que_jobs"
+        Que.execute "invalid SQL syntax"
+      end
+    end.should raise_error(PG::Error)
+
+    DB.table_exists?(:que_jobs).should be true
+  end
+
+  # In Ruby 1.9, it's impossible to tell inside an ensure block whether the
+  # currently executing thread has been killed.
+  unless RUBY_VERSION.start_with?('1.9')
+    it "should use transactions to protect its migrations from killed threads" do
+      q = Queue.new
+
+      t = Thread.new do
+        Que::Migrations.transaction do
+          Que.execute "DROP TABLE que_jobs"
+          q.push :go!
+          sleep
+        end
+      end
+
+      q.pop
+      t.kill
+      t.join
+
+      DB.table_exists?(:que_jobs).should be true
+    end
+  end
+end

data/spec/unit/pool_spec.rb

@@ -4,12 +4,15 @@ describe "Managing the Worker pool" do
   it "should log mode changes" do
     Que.mode = :sync
     Que.mode = :off
-    $logger.messages.should == ["[Que] Set mode to :sync", "[Que] Set mode to :off"]
-  end
 
-  it "Que.stop! should do nothing if there are no workers running" do
-    Que::Worker.workers.should be_empty
-    Que.stop!
+    $logger.messages.count.should be 2
+    m1, m2 = $logger.messages.map{|m| JSON.load(m)}
+
+    m1['event'].should == 'mode_change'
+    m1['value'].should == 'sync'
+
+    m2['event'].should == 'mode_change'
+    m2['value'].should == 'off'
   end
 
   describe "Que.mode = :sync" do
@@ -19,10 +22,6 @@ describe "Managing the Worker pool" do
       ArgsJob.queue(5, :testing => "synchronous").should be_an_instance_of ArgsJob
       $passed_args.should == [5, {'testing' => "synchronous"}]
       DB[:que_jobs].count.should be 0
-
-      $logger.messages.length.should be 2
-      $logger.messages[0].should == "[Que] Set mode to :sync"
-      $logger.messages[1].should =~ /\A\[Que\] Worked job in/
     end
 
     it "should not affect jobs that are queued with specific run_ats" do
@@ -31,11 +30,6 @@ describe "Managing the Worker pool" do
       ArgsJob.queue(5, :testing => "synchronous", :run_at => Time.now + 60)
       DB[:que_jobs].select_map(:job_class).should == ["ArgsJob"]
     end
-
-    it "then Que.stop! should do nothing" do
-      Que::Worker.workers.should be_empty
-      Que.stop!
-    end
   end
 
   describe "Que.mode = :async" do
@@ -44,7 +38,8 @@ describe "Managing the Worker pool" do
       Que.worker_count.should be 4
       sleep_until { Que::Worker.workers.all?(&:sleeping?) }
 
-      $logger.messages.should == ["[Que] Set mode to :async", "[Que] Set worker_count to 4"] + ["[Que] No jobs available..."] * 4
+      $logger.messages.map{|m| JSON.load(m).values_at('event', 'value')}.should ==
+        [['mode_change', 'async'], ['worker_count_change', '4']] + [['job_unavailable', nil]] * 4
     end
 
     it "should be done automatically when setting a worker count" do
@@ -53,7 +48,8 @@ describe "Managing the Worker pool" do
       Que.worker_count.should == 2
       sleep_until { Que::Worker.workers.all?(&:sleeping?) }
 
-      $logger.messages.should == ["[Que] Set mode to :async", "[Que] Set worker_count to 2"] + ["[Que] No jobs available..."] * 2
+      $logger.messages.map{|m| JSON.load(m).values_at('event', 'value')}.should ==
+        [['mode_change', 'async'], ['worker_count_change', '2']] + [['job_unavailable', nil]] * 2
     end
 
     it "should not affect the number of workers if a worker_count has already been set" do
@@ -62,7 +58,8 @@ describe "Managing the Worker pool" do
       Que.worker_count.should be 1
       sleep_until { Que::Worker.workers.all?(&:sleeping?) }
 
-      $logger.messages.should == ["[Que] Set mode to :async", "[Que] Set worker_count to 1", "[Que] No jobs available..."]
+      $logger.messages.map{|m| JSON.load(m).values_at('event', 'value')}.should ==
+        [['mode_change', 'async'], ['worker_count_change', '1'], ['job_unavailable', nil]]
     end
 
     it "then Que.worker_count = 0 should set the mode to :off" do
@@ -75,7 +72,8 @@ describe "Managing the Worker pool" do
       Que.worker_count.should == 0
       Que.mode.should == :off
 
-      $logger.messages.should == ["[Que] Set mode to :async", "[Que] Set worker_count to 4"] + ["[Que] No jobs available..."] * 4 + ["[Que] Set mode to :off", "[Que] Set worker_count to 0"]
+      $logger.messages.map{|m| JSON.load(m).values_at('event', 'value')}.should ==
+        [['mode_change', 'async'], ['worker_count_change', '4']] + [['job_unavailable', nil]] * 4 + [['mode_change', 'off'], ['worker_count_change', '0']]
     end
 
     it "then Que.worker_count = 2 should gracefully decrease the number of workers" do
@@ -94,7 +92,8 @@ describe "Managing the Worker pool" do
         worker.thread.status.should == false
       end
 
-      $logger.messages.should == ["[Que] Set mode to :async", "[Que] Set worker_count to 4"] + ["[Que] No jobs available..."] * 4 + ["[Que] Set worker_count to 2"]
+      $logger.messages.map{|m| JSON.load(m).values_at('event', 'value')}.should ==
+        [['mode_change', 'async'], ['worker_count_change', '4']] + [['job_unavailable', nil]] * 4 + [['worker_count_change', '2']]
     end
 
     it "then Que.worker_count = 6 should gracefully increase the number of workers" do
@@ -109,7 +108,8 @@ describe "Managing the Worker pool" do
 
       workers.should == Que::Worker.workers[0..3]
 
-      $logger.messages.should == ["[Que] Set mode to :async", "[Que] Set worker_count to 4"] + ["[Que] No jobs available..."] * 4 + ["[Que] Set worker_count to 6"] + ["[Que] No jobs available..."] * 2
+      $logger.messages.map{|m| JSON.load(m).values_at('event', 'value')}.should ==
+        [['mode_change', 'async'], ['worker_count_change', '4']] + [['job_unavailable', nil]] * 4 + [['worker_count_change', '6']] + [['job_unavailable', nil]] * 2
     end
 
     it "then Que.mode = :off should gracefully shut down workers" do
@@ -124,7 +124,8 @@ describe "Managing the Worker pool" do
       workers.count.should be 4
       workers.each { |worker| worker.thread.status.should be false }
 
-      $logger.messages.should == ["[Que] Set mode to :async", "[Que] Set worker_count to 4"] + ["[Que] No jobs available..."] * 4 + ["[Que] Set mode to :off", "[Que] Set worker_count to 0"]
+      $logger.messages.map{|m| JSON.load(m).values_at('event', 'value')}.should ==
+        [['mode_change', 'async'], ['worker_count_change', '4']] + [['job_unavailable', nil]] * 4 + [['mode_change', 'off'], ['worker_count_change', '0']]
     end
 
     it "then Que.wake! should wake up a single worker" do
@@ -181,24 +182,5 @@ describe "Managing the Worker pool" do
       Que::Job.queue
       sleep_until { DB[:que_jobs].count == 0 }
     end
-
-    it "then Que.stop! should interrupt all running jobs" do
-      begin
-        # Que.stop! can unpredictably affect connections, which may affect
-        # other tests, so use a new one.
-        pg = NEW_PG_CONNECTION.call
-        Que.connection = pg
-
-        BlockJob.queue
-        Que.mode = :async
-        $q1.pop
-        Que.stop!
-      ensure
-        if pg
-          # Closing the connection can raise a NullPointerException on JRuby.
-          pg.close rescue nil
-        end
-      end
-    end
   end
 end