que 0.0.1 → 0.1.0

data/que.gemspec

@@ -5,12 +5,12 @@ require 'que/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'que'
-  spec.version       = Que::VERSION
+  spec.version       = Que::Version
   spec.authors       = ["Chris Hanks"]
   spec.email         = ['christopher.m.hanks@gmail.com']
-  spec.description   = %q{Durable job queueing with PostgreSQL.}
-  spec.summary       = %q{Durable, efficient job queueing with PostgreSQL.}
-  spec.homepage      = ''
+  spec.description   = %q{A job queue that uses PostgreSQL's advisory locks for speed and reliability.}
+  spec.summary       = %q{A PostgreSQL-based Job Queue}
+  spec.homepage      = 'https://github.com/chanks/que'
   spec.license       = 'MIT'
 
   spec.files         = `git ls-files`.split($/)
@@ -23,6 +23,8 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 2.14.1'
   spec.add_development_dependency 'pry'
 
-  spec.add_dependency 'sequel'
-  spec.add_dependency 'pg'
+  spec.add_development_dependency 'sequel'
+  spec.add_development_dependency 'activerecord'
+  spec.add_development_dependency 'pg'
+  spec.add_development_dependency 'connection_pool'
 end

data/spec/adapters/active_record_spec.rb

@@ -0,0 +1,39 @@
+require 'spec_helper'
+require 'active_record'
+
+ActiveRecord::Base.establish_connection(QUE_URL)
+Que.connection = ActiveRecord
+QUE_ADAPTERS[:active_record] = Que.adapter
+
+describe "Que using the ActiveRecord adapter" do
+  before { Que.adapter = QUE_ADAPTERS[:active_record] }
+
+  it_behaves_like "a Que adapter"
+  it_behaves_like "a multithreaded Que adapter"
+
+  it "should use the same connection that ActiveRecord does" do
+    class ActiveRecordJob < Que::Job
+      def run
+        $pid1 = Que.execute("SELECT pg_backend_pid()").first['pg_backend_pid'].to_i
+        $pid2 = ActiveRecord::Base.connection.select_all("select pg_backend_pid()").rows.first.first.to_i
+      end
+    end
+
+    ActiveRecordJob.queue
+    Que::Job.work
+
+    $pid1.should == $pid2
+  end
+
+  it "should instantiate args as ActiveSupport::HashWithIndifferentAccess" do
+    ArgsJob.queue :param => 2
+    Que::Job.work
+    $passed_args.first[:param].should == 2
+    $passed_args.first.should be_an_instance_of ActiveSupport::HashWithIndifferentAccess
+  end
+
+  it "should support Rails' special extensions for times" do
+    Que::Job.queue :run_at => 1.minute.from_now
+    DB[:que_jobs].get(:run_at).should be_within(3).of Time.now + 60
+  end
+end

data/spec/adapters/connection_pool_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+require 'connection_pool'
+
+Que.connection = ConnectionPool.new &NEW_PG_CONNECTION
+QUE_ADAPTERS[:connection_pool] = Que.adapter
+
+describe "Que using the ConnectionPool adapter" do
+  before { Que.adapter = QUE_ADAPTERS[:connection_pool] }
+
+  it_behaves_like "a Que adapter"
+  it_behaves_like "a multithreaded Que adapter"
+end

data/spec/adapters/pg_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe "Que using a bare PG connection" do
+  it_behaves_like "a Que adapter"
+end

data/spec/adapters/sequel_spec.rb

@@ -0,0 +1,25 @@
+require 'spec_helper'
+
+Que.connection = SEQUEL_ADAPTER_DB = Sequel.connect(QUE_URL)
+QUE_ADAPTERS[:sequel] = Que.adapter
+
+describe "Que using the Sequel adapter" do
+  before { Que.adapter = QUE_ADAPTERS[:sequel] }
+
+  it_behaves_like "a Que adapter"
+  it_behaves_like "a multithreaded Que adapter"
+
+  it "should use the same connection that Sequel does" do
+    class SequelJob < Que::Job
+      def run
+        $pid1 = Que.execute("SELECT pg_backend_pid()").first['pg_backend_pid'].to_i
+        $pid2 = SEQUEL_ADAPTER_DB.get{pg_backend_pid{}}
+      end
+    end
+
+    SequelJob.queue
+    Que::Job.work
+
+    $pid1.should == $pid2
+  end
+end

data/spec/connection_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+
+describe Que do
+  it ".connection= with an unsupported connection should raise an error" do
+    proc{Que.connection = "ferret"}.should raise_error RuntimeError, /Que connection not recognized: "ferret"/
+  end
+
+  it ".adapter when no connection has been established should raise an error" do
+    Que.connection = nil
+    proc{Que.adapter}.should raise_error RuntimeError, /Que connection not established!/
+  end
+end

data/spec/helper_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe Que, 'helpers' do
+  it "should be able to drop and create the jobs table" do
+    DB.table_exists?(:que_jobs).should be true
+    Que.drop!
+    DB.table_exists?(:que_jobs).should be false
+    Que.execute "SET client_min_messages TO 'warning'" # Avoid annoying NOTICE messages.
+    Que.create!
+    DB.table_exists?(:que_jobs).should be true
+  end
+
+  it "should be able to clear the jobs table" do
+    DB[:que_jobs].insert :job_class => "Que::Job"
+    DB[:que_jobs].count.should be 1
+    Que.clear!
+    DB[:que_jobs].count.should be 0
+  end
+end

data/spec/pool_spec.rb

@@ -0,0 +1,116 @@
+require 'spec_helper'
+
+describe "Managing the Worker pool" do
+  it "should log mode changes" do
+    Que.mode = :off
+    $logger.messages.should == ["[Que] Set mode to :off"]
+  end
+
+  it "Que.mode = :sync should make jobs run in the same thread as they are queued" do
+    Que.mode = :sync
+
+    ArgsJob.queue(5, :testing => "synchronous").should be_an_instance_of ArgsJob
+    $passed_args.should == [5, {'testing' => "synchronous"}]
+    DB[:que_jobs].count.should be 0
+
+    $logger.messages.length.should be 2
+    $logger.messages[0].should == "[Que] Set mode to :sync"
+    $logger.messages[1].should =~ /\A\[Que\] Worked job in/
+  end
+
+  describe "Que.mode = :async" do
+    it "should spin up 4 workers" do
+      Que.mode = :async
+      workers = Que::Worker.workers
+      workers.count.should be 4
+      sleep_until { workers.all?(&:sleeping?) }
+    end
+
+    it "then Que.worker_count = 2 should gracefully decrease the number of workers" do
+      Que.mode = :async
+      workers = Que::Worker.workers.dup
+      workers.count.should be 4
+
+      Que.worker_count = 2
+      Que::Worker.workers.count.should be 2
+      sleep_until { Que::Worker.workers.all?(&:sleeping?) }
+
+      workers[0..1].should == Que::Worker.workers
+      workers[2..3].each do |worker|
+        worker.should be_an_instance_of Que::Worker
+        worker.thread.status.should == false
+      end
+    end
+
+    it "then Que.worker_count = 6 should gracefully increase the number of workers" do
+      Que.mode = :async
+      workers = Que::Worker.workers.dup
+      workers.count.should be 4
+
+      Que.worker_count = 6
+      Que::Worker.workers.count.should be 6
+      sleep_until { workers.all?(&:sleeping?) }
+
+      workers.should == Que::Worker.workers[0..3]
+    end
+
+    it "then Que.mode = :off should gracefully shut down workers" do
+      Que.mode = :async
+      workers = Que::Worker.workers.dup
+      workers.count.should be 4
+
+      Que.mode = :off
+      Que::Worker.workers.length.should be 0
+
+      workers.count.should be 4
+      workers.each { |worker| worker.thread.status.should be false }
+    end
+
+    it "then Que::Worker.wake! should wake up a single worker" do
+      Que.mode = :async
+      sleep_until { Que::Worker.workers.all? &:sleeping? }
+
+      BlockJob.queue
+      Que::Worker.wake!
+
+      $q1.pop
+      Que::Worker.workers.first.should be_working
+      Que::Worker.workers[1..3].each { |w| w.should be_sleeping }
+      DB[:que_jobs].count.should be 1
+      $q2.push nil
+
+      sleep_until { Que::Worker.workers.all? &:sleeping? }
+      DB[:que_jobs].count.should be 0
+    end
+
+    it "then Que::Worker.wake_all! should wake up all workers" do
+      # This spec requires at least four connections.
+      Que.adapter = QUE_ADAPTERS[:connection_pool]
+
+      Que.mode = :async
+      sleep_until { Que::Worker.workers.all? &:sleeping? }
+
+      4.times { BlockJob.queue }
+      Que::Worker.wake_all!
+      4.times { $q1.pop }
+
+      Que::Worker.workers.each{ |worker| worker.should be_working }
+      4.times { $q2.push nil }
+
+      sleep_until { Que::Worker.workers.all? &:sleeping? }
+      DB[:que_jobs].count.should be 0
+    end if QUE_ADAPTERS[:connection_pool]
+
+    it "should poke a worker every Que.sleep_period seconds" do
+      begin
+        Que.sleep_period = 0.001 # 1 ms
+        Que.mode = :async
+        sleep_until { Que::Worker.workers.all? &:sleeping? }
+        Que::Job.queue
+        sleep_until { DB[:que_jobs].count == 0 }
+      ensure
+        Que.sleep_period = nil
+      end
+    end
+  end
+end

data/spec/queue_spec.rb

@@ -0,0 +1,134 @@
+require 'spec_helper'
+
+describe Que::Job, '.queue' do
+  it "should be able to queue a job" do
+    DB[:que_jobs].count.should be 0
+    Que::Job.queue
+    DB[:que_jobs].count.should be 1
+
+    job = DB[:que_jobs].first
+    job[:priority].should be 1
+    job[:run_at].should be_within(3).of Time.now
+    job[:job_class].should == "Que::Job"
+    JSON.load(job[:args]).should == []
+  end
+
+  it "should be able to queue a job with arguments" do
+    DB[:que_jobs].count.should be 0
+    Que::Job.queue 1, 'two'
+    DB[:que_jobs].count.should be 1
+
+    job = DB[:que_jobs].first
+    job[:priority].should be 1
+    job[:run_at].should be_within(3).of Time.now
+    job[:job_class].should == "Que::Job"
+    JSON.load(job[:args]).should == [1, 'two']
+  end
+
+  it "should be able to queue a job with complex arguments" do
+    DB[:que_jobs].count.should be 0
+    Que::Job.queue 1, 'two', :string => "string",
+                             :integer => 5,
+                             :array => [1, "two", {:three => 3}],
+                             :hash => {:one => 1, :two => 'two', :three => [3]}
+
+    DB[:que_jobs].count.should be 1
+
+    job = DB[:que_jobs].first
+    job[:priority].should be 1
+    job[:run_at].should be_within(3).of Time.now
+    job[:job_class].should == "Que::Job"
+    JSON.load(job[:args]).should == [
+      1,
+      'two',
+      {
+        'string' => 'string',
+        'integer' => 5,
+        'array' => [1, "two", {"three" => 3}],
+        'hash' => {'one' => 1, 'two' => 'two', 'three' => [3]}
+      }
+    ]
+  end
+
+  it "should be able to queue a job with a specific time to run" do
+    DB[:que_jobs].count.should be 0
+    Que::Job.queue 1, :run_at => Time.now + 60
+    DB[:que_jobs].count.should be 1
+
+    job = DB[:que_jobs].first
+    job[:priority].should be 1
+    job[:run_at].should be_within(3).of Time.now + 60
+    job[:job_class].should == "Que::Job"
+    JSON.load(job[:args]).should == [1]
+  end
+
+  it "should be able to queue a job with a specific priority" do
+    DB[:que_jobs].count.should be 0
+    Que::Job.queue 1, :priority => 4
+    DB[:que_jobs].count.should be 1
+
+    job = DB[:que_jobs].first
+    job[:priority].should be 4
+    job[:run_at].should be_within(3).of Time.now
+    job[:job_class].should == "Que::Job"
+    JSON.load(job[:args]).should == [1]
+  end
+
+  it "should be able to queue a job with queueing options in addition to argument options" do
+    DB[:que_jobs].count.should be 0
+    Que::Job.queue 1, :string => "string", :run_at => Time.now + 60, :priority => 4
+    DB[:que_jobs].count.should be 1
+
+    job = DB[:que_jobs].first
+    job[:priority].should be 4
+    job[:run_at].should be_within(3).of Time.now + 60
+    job[:job_class].should == "Que::Job"
+    JSON.load(job[:args]).should == [1, {'string' => 'string'}]
+  end
+
+  it "should respect a default (but overridable) priority for the job class" do
+    class DefaultPriorityJob < Que::Job
+      @default_priority = 3
+    end
+
+    DB[:que_jobs].count.should be 0
+    DefaultPriorityJob.queue 1
+    DefaultPriorityJob.queue 1, :priority => 4
+    DB[:que_jobs].count.should be 2
+
+    first, second = DB[:que_jobs].order(:job_id).all
+
+    first[:priority].should be 3
+    first[:run_at].should be_within(3).of Time.now
+    first[:job_class].should == "DefaultPriorityJob"
+    JSON.load(first[:args]).should == [1]
+
+    second[:priority].should be 4
+    second[:run_at].should be_within(3).of Time.now
+    second[:job_class].should == "DefaultPriorityJob"
+    JSON.load(second[:args]).should == [1]
+  end
+
+  it "should respect a default (but overridable) run_at for the job class" do
+    class DefaultRunAtJob < Que::Job
+      @default_run_at = -> { Time.now + 60 }
+    end
+
+    DB[:que_jobs].count.should be 0
+    DefaultRunAtJob.queue 1
+    DefaultRunAtJob.queue 1, :run_at => Time.now + 30
+    DB[:que_jobs].count.should be 2
+
+    first, second = DB[:que_jobs].order(:job_id).all
+
+    first[:priority].should be 1
+    first[:run_at].should be_within(3).of Time.now + 60
+    first[:job_class].should == "DefaultRunAtJob"
+    JSON.load(first[:args]).should == [1]
+
+    second[:priority].should be 1
+    second[:run_at].should be_within(3).of Time.now + 30
+    second[:job_class].should == "DefaultRunAtJob"
+    JSON.load(second[:args]).should == [1]
+  end
+end

data/spec/spec_helper.rb

@@ -1,34 +1,57 @@
-require 'sequel'
 require 'que'
 
-DB = Sequel.connect "postgres://postgres:@localhost/que"
-
-DB.drop_table? :jobs
-DB.run <<-SQL
-  CREATE TABLE jobs
-  (
-    priority integer NOT NULL,
-    run_at timestamp with time zone NOT NULL DEFAULT now(),
-    job_id bigserial NOT NULL,
-    created_at timestamp with time zone NOT NULL DEFAULT now(),
-    type text NOT NULL,
-    args json NOT NULL DEFAULT '[]'::json,
-    data json NOT NULL DEFAULT '{}'::json,
-    CONSTRAINT jobs_pkey PRIMARY KEY (priority, run_at, job_id),
-    CONSTRAINT valid_priority CHECK (priority >= 1 AND priority <= 5)
-  );
-SQL
+Dir["./spec/support/**/*.rb"].sort.each &method(:require)
+
+QUE_URL = ENV["DATABASE_URL"] || "postgres://postgres:@localhost/que-test"
+
+
+
+# Handy proc to instantiate new PG connections:
+require 'uri'
+require 'pg'
+NEW_PG_CONNECTION = proc do
+  uri = URI.parse(QUE_URL)
+  PG::Connection.open :host     => uri.host,
+                      :user     => uri.user,
+                      :password => uri.password,
+                      :port     => uri.port || 5432,
+                      :dbname   => uri.path[1..-1]
+end
+
+
+
+# Adapters track information about their connections like which statements
+# have been prepared, and if Que.connection= is called before each spec, we're
+# constantly creating new adapters and losing that information, which is bad.
+# So instead, we hang onto a few adapters and assign them using Que.adapter=
+# as needed. The plain pg adapter is the default.
+
+# Also, let Que initialize the adapter itself, to make sure that the
+# recognition logic works. Similar code can be found in the adapter specs.
+Que.connection = NEW_PG_CONNECTION.call
+QUE_ADAPTERS = {:pg => Que.adapter}
+
+
+
+# We use Sequel to introspect the database in specs.
+require 'sequel'
+DB = Sequel.connect(QUE_URL)
+DB.drop_table? :que_jobs
+DB.run Que::SQL[:create_table]
 
 RSpec.configure do |config|
   config.before do
-    Que::Worker.state = :off
-    DB[:jobs].delete
+    DB[:que_jobs].delete
+    Que.adapter = QUE_ADAPTERS[:pg]
+    Que.mode = :off
+    Que.sleep_period = nil
+    $logger.messages.clear
   end
 end
 
-Que::Worker.state = :async # Boot up.
 
-# For use when debugging specs:
-# require 'logger'
-# Que.logger = Logger.new(STDOUT)
-# DB.loggers << Logger.new(STDOUT)
+
+# Set up a dummy logger.
+Que.logger = $logger = Object.new
+def $logger.messages;                   @messages ||= [];    end
+def $logger.method_missing(m, message); messages << message; end