queueing_rabbit 0.1.3 → 0.2.0

data/lib/queueing_rabbit/version.rb

@@ -1,3 +1,3 @@
 module QueueingRabbit
-  VERSION = "0.1.3"
+  VERSION = "0.2.0"
 end

data/lib/queueing_rabbit/worker.rb

@@ -69,14 +69,24 @@ module QueueingRabbit
     end
 
     def run_job(conn, job)
-      conn.open_channel(job.channel_options) do |channel, _|
-        conn.listen_queue(channel, job.queue_name, job.queue_options) do |args|
-          info "performing job #{job} with arguments #{args.inspect}"
-          job.perform(args)
+      QueueingRabbit.follow_job_requirements(job) do |_, _, queue|
+        conn.listen_queue(queue, job.listening_options) do |payload, metadata|
+          info "performing job #{job}"
+          invoke_job(job, payload, metadata)
         end
       end
     end
 
+    def invoke_job(job, payload, metadata)
+      if job.respond_to?(:perform)
+        job.perform(payload, metadata)
+      elsif job <= QueueingRabbit::AbstractJob
+        job.new(payload, metadata).perform
+      else
+        error "do not know how to perform job #{job}"
+      end
+    end
+
     def sync_stdio
       $stdout.sync = true
       $stderr.sync = true

data/lib/queueing_rabbit.rb

@@ -7,7 +7,11 @@ require "queueing_rabbit/client/callbacks"
 require "queueing_rabbit/client/amqp"
 require "queueing_rabbit/client/bunny"
 require "queueing_rabbit/extensions/new_relic"
+require "queueing_rabbit/extensions/retryable"
+require "queueing_rabbit/extensions/direct_exchange"
 require "queueing_rabbit/job"
+require "queueing_rabbit/jobs/abstract_job"
+require "queueing_rabbit/jobs/json_job"
 require "queueing_rabbit/worker"
 
 module QueueingRabbit
@@ -29,24 +33,35 @@ module QueueingRabbit
   def connection
     @connection ||= connect
   end
+  alias_method :conn, :connection
 
   def drop_connection
     @connection = nil
   end
 
-  def enqueue(job, arguments = {})
-    info "enqueueing job #{job} with arguments: #{arguments.inspect}."
+  def enqueue(job, payload = nil, options = {})
+    info "enqueueing job #{job}"
 
-    connection.open_channel(job.channel_options) do |c, _|
-      connection.define_queue(c, job.queue_name, job.queue_options)
-      connection.enqueue(c, job.queue_name, arguments)
-      c.close
+    follow_job_requirements(job) do |channel, exchange, _|
+      conn.enqueue(exchange, payload, options)
+      channel.close
     end
 
     true
   end
   alias_method :publish, :enqueue
 
+  def follow_job_requirements(job)
+    conn.open_channel(job.channel_options) do |ch, _|
+      conn.define_exchange(ch, job.exchange_name, job.exchange_options) do |ex|
+        conn.define_queue(ch, job.queue_name, job.queue_options) do |q|
+          conn.bind_queue(q, ex, job.binding_options) if job.bind_queue?
+          yield ch, ex, q if block_given?
+        end
+      end
+    end
+  end
+
   def queue_size(job)
     size = 0
     connection.open_channel(job.channel_options) do |c, _|

data/queueing_rabbit.gemspec

@@ -17,19 +17,17 @@ Gem::Specification.new do |gem|
   gem.extra_rdoc_files  = [ "LICENSE", "README.md" ]
   gem.rdoc_options      = ["--charset=UTF-8"]
 
-  gem.add_dependency "amqp",  ">= 0.9.0"
-  gem.add_dependency "bunny", ">= 0.9.0.pre7"
+  gem.add_dependency "amqp",  "~> 1.0.0"
+  gem.add_dependency "bunny", "~> 0.10.0"
   gem.add_dependency "rake",  ">= 0"
   gem.add_dependency "json",  ">= 0"
 
   gem.description   = <<description
     QueueingRabbit is a Ruby library providing convenient object-oriented syntax
-    for managing background jobs using AMQP. All jobs' argumets are serialized
-    to JSON and transfered to jobs using AMQP message payload. The library
-    implements amqp and bunny gems as adapters making it possible to use
-    synchronous publishing and asynchronous consuming, which might be useful for
-    Rails app running on non-EventMachine based application servers (i. e.
-    Passenger).
+    for managing background jobs using AMQP. The library implements amqp and
+    bunny gems as adapters making it possible to use synchronous publishing and
+    asynchronous consuming, which might be useful for Rails app running on
+    non-EventMachine based application servers (i. e. Passenger).
 
     Any Ruby class or Module can be transformed into QueueingRabbit's background
     job by including QueueingRabbit::Job module. It is also possible to inherit
@@ -37,14 +35,5 @@ Gem::Specification.new do |gem|
 
     The library is bundled with a Rake task which is capable of starting a
     worker processing a specified list of jobs.
-
-    To achieve the required simplicity the gem doesn't try to support all
-    features of AMQP protocol. It uses a restricted subset instead:
-
-    * Only a single direct exchange is used
-    * Every job is consumed using a separate channel
-    * All jobs are consumed with acknowledgements
-    * ACK is only sent to the broker if a job was processed successfully
-    * Currently all messages are published with persistent option
 description
 end

data/spec/integration/asynchronous_publishing_and_consuming_spec.rb

@@ -5,42 +5,60 @@ describe "Asynchronous publishing and consuming example" do
   include_context "Evented spec"
   include_context "StringIO logger"
 
-  let(:job) { PrintLineJob }
-
   before(:all) { QueueingRabbit.client = QueueingRabbit::Client::AMQP }
   after(:all) { QueueingRabbit.client = QueueingRabbit.default_client }
 
   context "basic consuming" do
+    let(:line) { "Hello, world!" }
     let(:connection) { QueueingRabbit.connection }
-    let(:worker) { QueueingRabbit::Worker.new(job.to_s) }
+    let(:job) { PrintLineJob }
     let(:io) { StringIO.new }
-    let(:line) { "Hello, world!" }
+    let(:worker) { QueueingRabbit::Worker.new(job.to_s) }
 
     before(:each) do
       QueueingRabbit.drop_connection
-      PrintLineJob.io = io
+    end
+
+    before do
+      job.io = io
     end
 
     it "processes enqueued jobs" do
       em {
         QueueingRabbit.connect
-        queue_size = nil
+        @queue_size = nil
+
+        def request_queue_size
+          connection.open_channel do |c, _|
+            connection.define_queue(c, job.queue_name, job.queue_options).status do |s, _|
+              @queue_size = s
+            end
+          end
+        end
 
         delayed(0.5) {
-          3.times { QueueingRabbit.enqueue(job, :line => line) }
+          3.times { job.enqueue(line) }
         }
 
-        delayed(1.0) { worker.work }
+        delayed(1.5) {
+          request_queue_size
+        }
 
         delayed(2.0) {
-          connection.open_channel do |c, _|
-            connection.define_queue(c, :print_line_job, job.queue_options).status do |s, _|
-              queue_size = s
-            end
-          end
+          @queue_size.should == 3
+        }
+
+        delayed(2.5) {
+          worker.work
+        }
+
+        delayed(3.5) {
+          request_queue_size
         }
 
-        done(3.0) { queue_size.should == 0 }
+        done(4.0) {
+          @queue_size.should be_zero
+        }
       }
     end
 
@@ -48,9 +66,11 @@ describe "Asynchronous publishing and consuming example" do
       em {
         QueueingRabbit.connect
 
-        delayed(0.5) { QueueingRabbit.enqueue(job, :line => line) }
+        delayed(0.5) { job.enqueue(line) }
 
-        delayed(1.0) { worker.work }
+        delayed(1.0) {
+          worker.work
+        }
 
         done(2.0) {
           io.string.should include(line)

data/spec/integration/asynchronous_publishing_and_consuming_with_retries_spec.rb

@@ -0,0 +1,103 @@
+require 'spec_helper'
+require 'integration/jobs/print_line_job'
+
+describe "Asynchronous publishing and consuming with retries" do
+  include_context "Evented spec"
+  include_context "StringIO logger"
+
+  before(:all) { QueueingRabbit.client = QueueingRabbit::Client::AMQP }
+  after(:all) { QueueingRabbit.client = QueueingRabbit.default_client }
+
+  context "basic consuming" do
+    let(:line) { "Hello, world!" }
+    let(:connection) { QueueingRabbit.connection }
+    let(:job) {
+      Class.new(QueueingRabbit::AbstractJob) do
+        class << self
+          attr_accessor :io, :times_executed
+        end
+
+        include QueueingRabbit::JobExtensions::Retryable
+        queue 'retryable_print_line_job'
+
+        def perform
+          self.class.times_executed = self.class.times_executed.to_i + 1
+          if retries > 2
+            self.class.io << 'Hello, world!'
+          else
+            retry_upto(3)
+          end
+        end
+      end
+    }
+    let(:job_name) { 'RetryablePrintLineJob' }
+    let(:io) { StringIO.new }
+    let(:worker) { QueueingRabbit::Worker.new(job_name) }
+
+    before(:each) do
+      QueueingRabbit.drop_connection
+    end
+
+    before do
+      job.io = io
+      stub_const(job_name, job)
+    end
+
+    it "processes enqueued jobs" do
+      em {
+        QueueingRabbit.connect
+        @queue_size = nil
+
+        def request_queue_size
+          connection.open_channel do |c, _|
+            connection.define_queue(c, job.queue_name, job.queue_options).status do |s, _|
+              @queue_size = s
+            end
+          end
+        end
+
+        delayed(0.5) {
+          3.times { job.enqueue(line) }
+        }
+
+        delayed(1.5) {
+          request_queue_size
+        }
+
+        delayed(2.0) {
+          @queue_size.should == 3
+        }
+
+        delayed(2.5) {
+          worker.work
+        }
+
+        delayed(3.5) {
+          request_queue_size
+        }
+
+        done(4.0) {
+          @queue_size.should be_zero
+          job.times_executed.should == 12
+        }
+      }
+    end
+
+    it "actually outputs the line" do
+      em {
+        QueueingRabbit.connect
+
+        delayed(0.5) { job.enqueue(line) }
+
+        delayed(1.0) {
+          worker.work
+        }
+
+        done(2.0) {
+          io.string.should include(line)
+        }
+      }
+    end
+  end
+end
+

data/spec/integration/direct_exchange_asynchronous_publishing_and_consuming_spec.rb

@@ -0,0 +1,93 @@
+require 'spec_helper'
+require 'integration/jobs/print_line_job'
+
+describe "Asynchronous publishing and consuming using direct exchange" do
+  include_context "Evented spec"
+  include_context "StringIO logger"
+
+  before(:all) { QueueingRabbit.client = QueueingRabbit::Client::AMQP }
+  after(:all) { QueueingRabbit.client = QueueingRabbit.default_client }
+
+  context "basic consuming" do
+    let(:line) { "Hello, world!" }
+    let(:connection) { QueueingRabbit.connection }
+    let(:job) {
+      Class.new(PrintLineJob) do
+        include QueueingRabbit::JobExtensions::DirectExchange
+        queue 'print_line_job_direct_exchange'
+        exchange 'queueing_rabbit.test', :type => :direct
+      end
+    }
+    let(:job_name) { 'PrintLineDirectExchangeJob' }
+    let(:io) { StringIO.new }
+    let(:worker) { QueueingRabbit::Worker.new(job_name) }
+
+    before do
+      stub_const(job_name, job)
+    end
+
+    before(:each) do
+      QueueingRabbit.drop_connection
+    end
+
+    before do
+      job.io = io
+    end
+
+    it "processes enqueued jobs" do
+      em {
+        QueueingRabbit.connect
+        @queue_size = nil
+
+        def request_queue_size
+          connection.open_channel do |c, _|
+            connection.define_queue(c, job.queue_name, job.queue_options).status do |s, _|
+              @queue_size = s
+            end
+          end
+        end
+
+        delayed(0.5) {
+          3.times { job.enqueue(line) }
+        }
+
+        delayed(1.5) {
+          request_queue_size
+        }
+
+        delayed(2.0) {
+          @queue_size.should == 3
+        }
+
+        delayed(2.5) {
+          worker.work
+        }
+
+        delayed(3.5) {
+          request_queue_size
+        }
+
+        done(4.0) {
+          @queue_size.should be_zero
+        }
+      }
+    end
+
+    it "actually outputs the line" do
+      em {
+        QueueingRabbit.connect
+
+        delayed(0.5) { job.enqueue(line) }
+
+        delayed(1.0) {
+          worker.work
+        }
+
+        done(2.0) {
+          io.string.should include(line)
+        }
+      }
+    end
+  end
+end
+

data/spec/integration/jobs/print_line_job.rb

@@ -1,17 +1,11 @@
-class PrintLineJob
-  extend QueueingRabbit::Job
-
+class PrintLineJob < QueueingRabbit::AbstractJob
   class << self
-    attr_writer :io
+    attr_accessor :io
   end
 
-  queue :print_line_job, :durable => true
-
-  def self.perform(arguments = {})
-    self.io.puts arguments[:line]
-  end
+  queue :print_line_job
 
-  def self.io
-    @io ||= STDOUT
+  def perform
+    self.class.io.puts payload
   end
 end

data/spec/integration/json_job_asynchronous_publishing_and_consuming_spec.rb

@@ -0,0 +1,99 @@
+require 'spec_helper'
+require 'integration/jobs/print_line_job'
+
+describe "Asynchronous publishing and consuming with JSON serialization" do
+  include_context "Evented spec"
+  include_context "StringIO logger"
+
+  before(:all) { QueueingRabbit.client = QueueingRabbit::Client::AMQP }
+  after(:all) { QueueingRabbit.client = QueueingRabbit.default_client }
+
+  context "basic consuming" do
+    let(:line) { "Hello, world!" }
+    let(:connection) { QueueingRabbit.connection }
+    let(:job) {
+      Class.new(QueueingRabbit::JSONJob) do
+        class << self
+          attr_accessor :io
+        end
+
+        def perform
+          self.class.io.puts arguments[:line]
+        end
+      end
+
+    }
+    let(:job_name) { 'PrintLineFromJSONJob' }
+    let(:io) { StringIO.new }
+    let(:worker) { QueueingRabbit::Worker.new(job_name) }
+
+    before do
+      stub_const(job_name, job)
+      job.io = io
+    end
+
+    before(:each) do
+      QueueingRabbit.drop_connection
+    end
+
+    before do
+      job.io = io
+    end
+
+    it "processes enqueued jobs" do
+      em {
+        QueueingRabbit.connect
+        @queue_size = nil
+
+        def request_queue_size
+          connection.open_channel do |c, _|
+            connection.define_queue(c, job.queue_name, job.queue_options).status do |s, _|
+              @queue_size = s
+            end
+          end
+        end
+
+        delayed(0.5) {
+          3.times { job.enqueue(:line => line) }
+        }
+
+        delayed(1.5) {
+          request_queue_size
+        }
+
+        delayed(2.0) {
+          @queue_size.should == 3
+        }
+
+        delayed(2.5) {
+          worker.work
+        }
+
+        delayed(3.5) {
+          request_queue_size
+        }
+
+        done(4.0) {
+          @queue_size.should be_zero
+        }
+      }
+    end
+
+    it "actually outputs the line" do
+      em {
+        QueueingRabbit.connect
+
+        delayed(0.5) { job.enqueue(:line => line) }
+
+        delayed(1.0) {
+          worker.work
+        }
+
+        done(2.0) {
+          io.string.should include(line)
+        }
+      }
+    end
+  end
+end
+